This commit is contained in:
Nebel 2023-12-19 21:16:08 +09:00
parent 59c28ca0cd
commit fef40d6b45
Signed by: nebel
GPG Key ID: 79807D08C6EF6460
11 changed files with 691 additions and 0 deletions

238
browser.go Normal file
View File

@ -0,0 +1,238 @@
package main
import (
"encoding/base64"
"encoding/json"
"errors"
"log"
"net/http"
"net/url"
"strings"
"github.com/playwright-community/playwright-go"
)
type rawImageFile struct {
url string
blocks []map[string]int
width *int
height *int
}
type file struct {
contentType string
data []byte
}
type browser struct {
db *database
pw *playwright.Playwright
browser *playwright.Browser
}
func installBrowser() error {
opts := playwright.RunOptions{
Browsers: []string{"chromium"},
Verbose: true,
}
return playwright.Install(&opts)
}
type runOptions struct {
db *database
headless bool
}
func runBrowser(options runOptions) *browser {
pw, err := playwright.Run()
if err != nil {
log.Fatal(err)
}
launchOptions := playwright.BrowserTypeLaunchOptions{
Headless: playwright.Bool(options.headless),
}
b, err := pw.Chromium.Launch(launchOptions)
if err != nil {
log.Fatal(err)
}
browser := browser{
db: options.db,
pw: pw,
browser: &b,
}
return &browser
}
func (b browser) stop() error {
if b.browser != nil {
(*b.browser).Close()
}
return b.pw.Stop()
}
func (b browser) newContext() (playwright.BrowserContext, error) {
browser := *b.browser
return browser.NewContext()
}
func (b browser) loadBrowserContext(platform string) (playwright.BrowserContext, error) {
var secrets string
row := b.db.QueryRow("select secrets from platforms where name = $1", platform)
if err := row.Scan(&secrets); err != nil {
return nil, err
}
var storageState playwright.OptionalStorageState
if err := json.Unmarshal([]byte(secrets), &storageState); err != nil {
return nil, err
}
browserNewContextOptions := playwright.BrowserNewContextOptions{
StorageState: &storageState,
}
ctx, err := (*b.browser).NewContext(browserNewContextOptions)
if err != nil {
return nil, err
}
return ctx, nil
}
func (b browser) saveBrowserContext(platform string, ctx playwright.BrowserContext) error {
storageState, err := ctx.StorageState()
if err != nil {
return err
}
secrets, err := json.Marshal(storageState)
if err != nil {
return err
}
if _, err := b.db.Exec("update platforms set secrets = $1 where name = $2", secrets, platform); err != nil {
return err
}
return nil
}
func parseDataURL(dataURL string) (*file, error) {
if !strings.HasPrefix(dataURL, "data:") {
return nil, errors.New("not data scheme")
}
parts := strings.SplitN(dataURL, ",", 2)
if len(parts) != 2 {
return nil, errors.New("invalid data URL")
}
raw, err := url.PathUnescape(parts[1])
if err != nil {
return nil, err
}
var data []byte
if strings.HasSuffix(parts[0], ";base64") {
data, err = base64.StdEncoding.DecodeString(raw)
if err != nil {
return nil, err
}
} else {
data = []byte(raw)
}
file := file{
contentType: http.DetectContentType(data),
data: data,
}
return &file, nil
}
var draw = `
async function drawImage(imageFile) {
const canvas = Object.assign(document.createElement("canvas"), {
width: imageFile.width,
height: imageFile.height,
});
const image = (await new Promise((resolve) => {
Object.assign(new Image(), {
crossOrigin: "use-credentials",
src: imageFile.url,
onload() {
resolve(this);
},
});
}));
const ctx = canvas.getContext("2d");
for (const q of imageFile.blocks) {
ctx.drawImage(image, q.destX, q.destY, q.width, q.height, q.srcX, q.srcY, q.width, q.height);
}
const dataUrl = canvas.toDataURL();
return dataUrl;
}
`
var fetch = `
async function fetchImage(imageFile) {
const res = await fetch(imageFile.url);
const blob = await res.blob();
const dataUrl = await new Promise((resolve, reject) => {
const fileReader = Object.assign(new FileReader(), {
onload() {
resolve(this.result);
},
onerror(e) {
const error = new Error(` + "`${e.type}: ${e.message}`" + `);
reject(error);
},
});
fileReader.readAsDataURL(blob);
});
return dataUrl;
}
`
func (b browser) drawImage(page playwright.Page, imageFile rawImageFile) (*file, error) {
if len(imageFile.blocks) > 0 {
dataURL, err := page.Evaluate(draw, imageFile)
if err != nil {
return nil, err
}
return parseDataURL(dataURL.(string))
}
if strings.HasPrefix(imageFile.url, "blob:") {
dataURL, err := page.Evaluate(fetch, imageFile)
if err != nil {
return nil, err
}
return parseDataURL(dataURL.(string))
}
res, err := page.Context().Request().Get(imageFile.url)
if err != nil {
return nil, err
}
data, err := res.Body()
if err != nil {
return nil, err
}
file := file{
contentType: http.DetectContentType(data),
data: data,
}
return &file, nil
}

49
database.go Normal file
View File

@ -0,0 +1,49 @@
package main
import (
"database/sql"
"embed"
"log"
"net/url"
"os"
"path/filepath"
"github.com/amacneil/dbmate/v2/pkg/dbmate"
_ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite"
_ "github.com/mattn/go-sqlite3"
)
//go:embed migrations/*.sql
var fs embed.FS
type database struct {
*sql.DB
}
func newDatabase(path string) *database {
url, err := url.Parse("sqlite3://")
if err != nil {
log.Fatal(err)
}
url.Path, err = filepath.Abs(path)
if err != nil {
log.Fatal(err)
}
dbmate := dbmate.New(url)
dbmate.AutoDumpSchema = false
dbmate.FS = fs
dbmate.MigrationsDir = []string{"migrations"}
dbmate.CreateAndMigrate()
if err := os.Chmod(url.Path, 0600); err != nil {
log.Fatal(err)
}
db, err := sql.Open("sqlite3", url.Path)
if err != nil {
log.Fatal(err)
}
return &database{db}
}

21
go.mod Normal file
View File

@ -0,0 +1,21 @@
module git.fogtype.com/nebel/gadl
go 1.21.5
require (
github.com/amacneil/dbmate/v2 v2.9.0
github.com/mattn/go-sqlite3 v1.14.19
github.com/playwright-community/playwright-go v0.3900.1
github.com/urfave/cli/v2 v2.26.0
)
require (
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
github.com/go-jose/go-jose/v3 v3.0.1 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
go.uber.org/multierr v1.11.0 // indirect
)

49
go.sum Normal file
View File

@ -0,0 +1,49 @@
github.com/amacneil/dbmate/v2 v2.9.0 h1:uXBlYKEQJL2gwXdiSlJLcXpgpibeak3OY0NN0oM/TCM=
github.com/amacneil/dbmate/v2 v2.9.0/go.mod h1:ygYXKjaOsIUnIZa/jfyosyfN2BXwwRY8uDGGTvQCADQ=
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/playwright-community/playwright-go v0.3900.1 h1:8BkmDxVzLTp3USQ50EyXJSXcz0XDMwNP5y29lHIZ9Fc=
github.com/playwright-community/playwright-go v0.3900.1/go.mod h1:mbNzMqt04IVRdhVfXWqmCxd81gCdL3BA5hj6/pVAIqM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI=
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 h1:qXafrlZL1WsJW5OokjraLLRURHiw0OzKHD/RNdspp4w=
github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQxz6hGoNFBC4nIx+CxZhI3nne5RmIOlT/MXcSD4=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

140
library.go Normal file
View File

@ -0,0 +1,140 @@
package main
import (
"encoding/json"
)
type book struct {
ID int `json:"id"`
Platform string `json:"platform"`
ReaderURL string `json:"readerUrl"`
Title string `json:"title"`
Authors []string `json:"authors"`
}
type library struct {
db *database
}
type newLibraryOptions struct {
db *database
}
func newLibrary(options newLibraryOptions) *library {
return &library{options.db}
}
func (lib *library) add(b book) error {
authors, err := json.Marshal(b.Authors)
if err != nil {
return err
}
_, err = lib.db.Exec(`
insert into books(
platform_id,
reader_url,
title,
authors)
values((select id from platforms where name = $1), $2, $3, $4)
on conflict(reader_url)
do update set title = excluded.title, authors = excluded.authors
`,
b.Platform,
b.ReaderURL,
b.Title,
authors,
)
return err
}
func (lib *library) delete(id string) error {
_, err := lib.db.Exec(`delete from books where id = $1`, id)
return err
}
func (lib *library) get(readerURLOrBookID string) (*book, error) {
row := lib.db.QueryRow(`
select
books.id,
platforms.name,
books.reader_url,
books.title,
books.authors
from books
left join platforms
on books.platform_id = platforms.id
where books.reader_url = $1 or books.id = $1
`,
readerURLOrBookID,
)
var (
b book
authors []byte
)
err := row.Scan(
&b.ID,
&b.Platform,
&b.ReaderURL,
&b.Title,
&authors,
)
if err != nil {
return nil, err
}
err = json.Unmarshal(authors, &b.Authors)
return &b, err
}
func (lib *library) getBooks() (*[]book, error) {
rows, err := lib.db.Query(`
select
books.id,
platforms.name,
books.reader_url,
books.title,
books.authors
from books
left join platforms
on books.platform_id = platforms.id
`)
if err != nil {
return nil, err
}
bs := &[]book{}
for rows.Next() {
var (
b book
authors []byte
)
if err := rows.Scan(
&b.ID,
&b.Platform,
&b.ReaderURL,
&b.Title,
&authors,
); err != nil {
return nil, err
}
if err = json.Unmarshal(authors, &b.Authors); err != nil {
return nil, err
}
*bs = append(*bs, b)
}
return bs, err
}

170
main.go Normal file
View File

@ -0,0 +1,170 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
_ "github.com/mattn/go-sqlite3"
"github.com/urfave/cli/v2"
)
var version = "v1.0.0"
/*
TODO
- platform
- login
- logout
- pull
- download
*/
func main() {
flags := []cli.Flag{
&cli.StringFlag{
Name: "db",
Usage: "database file path",
Value: "gadl.db",
},
&cli.StringFlag{
Name: "platform",
Usage: "platform type",
Value: "google-play-books",
},
}
app := &cli.App{
Name: "gadl",
Usage: "Manga Downloader",
Version: version,
Flags: flags,
Commands: cli.Commands{
&cli.Command{
Name: "install",
Usage: "install system dependencies for browser",
Flags: flags,
Action: func(c *cli.Context) error {
return installBrowser()
},
},
&cli.Command{
Name: "login",
Flags: flags,
Action: func(c *cli.Context) error {
db := newDatabase(c.String("db"))
browser := runBrowser(runOptions{db: db, headless: false})
platform := c.String("platform")
_, err := browser.loadBrowserContext(platform)
if err != nil {
log.Fatal(err)
}
return browser.stop()
},
},
&cli.Command{
Name: "add",
Flags: flags,
Action: func(c *cli.Context) error {
db := newDatabase(c.String("db"))
library := newLibrary(newLibraryOptions{db})
for _, url := range c.Args().Slice() {
b := book{
// TODO: url to platform
Platform: c.String("platform"),
ReaderURL: url,
}
if err := library.add(b); err != nil {
log.Fatal(err)
}
}
return nil
},
},
&cli.Command{
Name: "delete",
Flags: flags,
Action: func(c *cli.Context) error {
db := newDatabase(c.String("db"))
library := newLibrary(newLibraryOptions{db})
for _, id := range c.Args().Slice() {
if err := library.delete(id); err != nil {
log.Fatal(err)
}
}
return nil
},
},
&cli.Command{
Name: "list",
Flags: flags,
Action: func(c *cli.Context) error {
db := newDatabase(c.String("db"))
library := newLibrary(newLibraryOptions{db})
bs, err := library.getBooks()
if err != nil {
log.Fatal(err)
}
for _, b := range *bs {
json, err := json.Marshal(&b)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(json))
}
return nil
},
},
&cli.Command{
Name: "view",
Flags: flags,
Action: func(c *cli.Context) error {
db := newDatabase(c.String("db"))
library := newLibrary(newLibraryOptions{db})
for _, id := range c.Args().Slice() {
b, err := library.get(id)
if err != nil {
log.Fatal(err)
}
json, err := json.Marshal(&b)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(json))
}
return nil
},
},
&cli.Command{
Name: "download",
Flags: flags,
Action: func(c *cli.Context) error {
newDatabase(c.String("db"))
return nil
},
},
},
}
if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}

12
migrations/0_init.sql Normal file
View File

@ -0,0 +1,12 @@
-- migrate:up
create table if not exists migrations (
id integer primary key
);
insert into schema_migrations(version)
select id
from migrations;
drop table if exists migrations;
-- migrate:down

View File

@ -1,3 +1,4 @@
-- migrate:up
create table platforms (
id integer primary key autoincrement,
name text unique not null,
@ -14,3 +15,5 @@ create table books (
insert into platforms(name) values
('dmm-books'),
('google-play-books');
-- migrate:down

View File

@ -1,2 +1,5 @@
-- migrate:up
alter table books add column title text not null default '';
alter table books add column authors json not null default '[]';
-- migrate:down

View File

@ -1,2 +1,5 @@
-- migrate:up
insert into platforms(name) values
('fanza-doujin');
-- migrate:down

View File

@ -1,2 +1,5 @@
-- migrate:up
insert into platforms(name) values
('dlsite-maniax');
-- migrate:down