diff --git a/browser.go b/browser.go new file mode 100644 index 0000000..b637d81 --- /dev/null +++ b/browser.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "errors" + "log" + "net/http" + "net/url" + "strings" + + "github.com/playwright-community/playwright-go" +) + +type rawImageFile struct { + url string + blocks []map[string]int + width *int + height *int +} + +type file struct { + contentType string + data []byte +} + +type browser struct { + db *database + pw *playwright.Playwright + browser *playwright.Browser +} + +func installBrowser() error { + opts := playwright.RunOptions{ + Browsers: []string{"chromium"}, + Verbose: true, + } + + return playwright.Install(&opts) +} + +type runOptions struct { + db *database + headless bool +} + +func runBrowser(options runOptions) *browser { + pw, err := playwright.Run() + if err != nil { + log.Fatal(err) + } + + launchOptions := playwright.BrowserTypeLaunchOptions{ + Headless: playwright.Bool(options.headless), + } + + b, err := pw.Chromium.Launch(launchOptions) + if err != nil { + log.Fatal(err) + } + + browser := browser{ + db: options.db, + pw: pw, + browser: &b, + } + + return &browser +} + +func (b browser) stop() error { + if b.browser != nil { + (*b.browser).Close() + } + + return b.pw.Stop() +} + +func (b browser) newContext() (playwright.BrowserContext, error) { + browser := *b.browser + + return browser.NewContext() +} + +func (b browser) loadBrowserContext(platform string) (playwright.BrowserContext, error) { + var secrets string + row := b.db.QueryRow("select secrets from platforms where name = $1", platform) + if err := row.Scan(&secrets); err != nil { + return nil, err + } + + var storageState playwright.OptionalStorageState + if err := json.Unmarshal([]byte(secrets), &storageState); err != nil { + return nil, err + } + + browserNewContextOptions := playwright.BrowserNewContextOptions{ + StorageState: &storageState, + } + + ctx, err := (*b.browser).NewContext(browserNewContextOptions) + if err != nil { + return nil, err + } + + return ctx, nil +} + +func (b browser) saveBrowserContext(platform string, ctx playwright.BrowserContext) error { + storageState, err := ctx.StorageState() + if err != nil { + return err + } + + secrets, err := json.Marshal(storageState) + if err != nil { + return err + } + + if _, err := b.db.Exec("update platforms set secrets = $1 where name = $2", secrets, platform); err != nil { + return err + } + + return nil +} + +func parseDataURL(dataURL string) (*file, error) { + if !strings.HasPrefix(dataURL, "data:") { + return nil, errors.New("not data scheme") + } + parts := strings.SplitN(dataURL, ",", 2) + if len(parts) != 2 { + return nil, errors.New("invalid data URL") + } + raw, err := url.PathUnescape(parts[1]) + if err != nil { + return nil, err + } + + var data []byte + + if strings.HasSuffix(parts[0], ";base64") { + data, err = base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, err + } + } else { + data = []byte(raw) + } + + file := file{ + contentType: http.DetectContentType(data), + data: data, + } + + return &file, nil +} + +var draw = ` +async function drawImage(imageFile) { + const canvas = Object.assign(document.createElement("canvas"), { + width: imageFile.width, + height: imageFile.height, + }); + const image = (await new Promise((resolve) => { + Object.assign(new Image(), { + crossOrigin: "use-credentials", + src: imageFile.url, + onload() { + resolve(this); + }, + }); + })); + const ctx = canvas.getContext("2d"); + for (const q of imageFile.blocks) { + ctx.drawImage(image, q.destX, q.destY, q.width, q.height, q.srcX, q.srcY, q.width, q.height); + } + const dataUrl = canvas.toDataURL(); + return dataUrl; +} +` + +var fetch = ` +async function fetchImage(imageFile) { + const res = await fetch(imageFile.url); + const blob = await res.blob(); + const dataUrl = await new Promise((resolve, reject) => { + const fileReader = Object.assign(new FileReader(), { + onload() { + resolve(this.result); + }, + onerror(e) { + const error = new Error(` + "`${e.type}: ${e.message}`" + `); + reject(error); + }, + }); + fileReader.readAsDataURL(blob); + }); + return dataUrl; +} +` + +func (b browser) drawImage(page playwright.Page, imageFile rawImageFile) (*file, error) { + if len(imageFile.blocks) > 0 { + dataURL, err := page.Evaluate(draw, imageFile) + if err != nil { + return nil, err + } + + return parseDataURL(dataURL.(string)) + } + + if strings.HasPrefix(imageFile.url, "blob:") { + dataURL, err := page.Evaluate(fetch, imageFile) + if err != nil { + return nil, err + } + + return parseDataURL(dataURL.(string)) + } + + res, err := page.Context().Request().Get(imageFile.url) + if err != nil { + return nil, err + } + + data, err := res.Body() + if err != nil { + return nil, err + } + + file := file{ + contentType: http.DetectContentType(data), + data: data, + } + + return &file, nil +} diff --git a/database.go b/database.go new file mode 100644 index 0000000..70f2ce8 --- /dev/null +++ b/database.go @@ -0,0 +1,49 @@ +package main + +import ( + "database/sql" + "embed" + "log" + "net/url" + "os" + "path/filepath" + + "github.com/amacneil/dbmate/v2/pkg/dbmate" + _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" + _ "github.com/mattn/go-sqlite3" +) + +//go:embed migrations/*.sql +var fs embed.FS + +type database struct { + *sql.DB +} + +func newDatabase(path string) *database { + url, err := url.Parse("sqlite3://") + if err != nil { + log.Fatal(err) + } + url.Path, err = filepath.Abs(path) + if err != nil { + log.Fatal(err) + } + + dbmate := dbmate.New(url) + dbmate.AutoDumpSchema = false + dbmate.FS = fs + dbmate.MigrationsDir = []string{"migrations"} + dbmate.CreateAndMigrate() + + if err := os.Chmod(url.Path, 0600); err != nil { + log.Fatal(err) + } + + db, err := sql.Open("sqlite3", url.Path) + if err != nil { + log.Fatal(err) + } + + return &database{db} +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..279b8f2 --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module git.fogtype.com/nebel/gadl + +go 1.21.5 + +require ( + github.com/amacneil/dbmate/v2 v2.9.0 + github.com/mattn/go-sqlite3 v1.14.19 + github.com/playwright-community/playwright-go v0.3900.1 + github.com/urfave/cli/v2 v2.26.0 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect + github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect + go.uber.org/multierr v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5736b7b --- /dev/null +++ b/go.sum @@ -0,0 +1,49 @@ +github.com/amacneil/dbmate/v2 v2.9.0 h1:uXBlYKEQJL2gwXdiSlJLcXpgpibeak3OY0NN0oM/TCM= +github.com/amacneil/dbmate/v2 v2.9.0/go.mod h1:ygYXKjaOsIUnIZa/jfyosyfN2BXwwRY8uDGGTvQCADQ= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= +github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/playwright-community/playwright-go v0.3900.1 h1:8BkmDxVzLTp3USQ50EyXJSXcz0XDMwNP5y29lHIZ9Fc= +github.com/playwright-community/playwright-go v0.3900.1/go.mod h1:mbNzMqt04IVRdhVfXWqmCxd81gCdL3BA5hj6/pVAIqM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= +github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= +github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 h1:qXafrlZL1WsJW5OokjraLLRURHiw0OzKHD/RNdspp4w= +github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQxz6hGoNFBC4nIx+CxZhI3nne5RmIOlT/MXcSD4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/library.go b/library.go new file mode 100644 index 0000000..8ae57ae --- /dev/null +++ b/library.go @@ -0,0 +1,95 @@ +package main + +import ( + "encoding/json" +) + +type book struct { + ID int `json:"id"` + Platform string `json:"platform"` + ReaderURL string `json:"readerUrl"` + Title string `json:"title"` + Authors []string `json:"authors"` +} + +type library struct { + db *database +} + +type newLibraryOptions struct { + db *database +} + +func newLibrary(options newLibraryOptions) *library { + return &library{options.db} +} + +func (lib *library) add(b book) error { + authors, err := json.Marshal(b.Authors) + if err != nil { + return err + } + + _, err = lib.db.Exec(` + insert into books( + platform_id, + reader_url, + title, + authors) + values((select id from platforms where name = $1), $2, $3, $4) + on conflict(reader_url) + do update set title = excluded.title, authors = excluded.authors +`, + + b.Platform, + b.ReaderURL, + b.Title, + authors, + ) + + return err +} + +func (lib *library) delete(id string) error { + _, err := lib.db.Exec(`delete from books where id = $1`, id) + + return err +} + +func (lib *library) get(readerURLOrBookID string) (*book, error) { + row := lib.db.QueryRow(` + select + books.id, + platforms.name, + books.reader_url, + books.title, + books.authors + from books + left join platforms + on books.platform_id = platforms.id + where books.reader_url = $1 or books.id = $1 +`, + readerURLOrBookID, + ) + + var ( + b book + authors []byte + ) + + err := row.Scan( + &b.ID, + &b.Platform, + &b.ReaderURL, + &b.Title, + &authors, + ) + + if err != nil { + return nil, err + } + + err = json.Unmarshal(authors, &b.Authors) + + return &b, err +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..4bcb1a4 --- /dev/null +++ b/main.go @@ -0,0 +1,127 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + + _ "github.com/mattn/go-sqlite3" + "github.com/urfave/cli/v2" +) + +var version = "v1.0.0" + +/* +TODO + +- library + - add + - list - getBooks +- platform + - login + - logout + - pull + - download + +*/ + +func main() { + flags := []cli.Flag{ + &cli.StringFlag{ + Name: "db", + Usage: "database file path", + Value: "gadl.db", + }, + &cli.StringFlag{ + Name: "platform", + Usage: "platform type", + Value: "google-play-books", + }, + } + + app := &cli.App{ + Name: "gadl", + Usage: "Manga Downloader", + Version: version, + Flags: flags, + Commands: cli.Commands{ + &cli.Command{ + Name: "install", + Usage: "install system dependencies for browser", + Flags: flags, + Action: func(c *cli.Context) error { + return installBrowser() + }, + }, + &cli.Command{ + Name: "login", + Flags: flags, + Action: func(c *cli.Context) error { + db := newDatabase(c.String("db")) + browser := runBrowser(runOptions{db: db, headless: false}) + platform := c.String("platform") + _, err := browser.loadBrowserContext(platform) + if err != nil { + log.Fatal(err) + } + + return browser.stop() + }, + }, + &cli.Command{ + Name: "delete", + Flags: flags, + Action: func(c *cli.Context) error { + db := newDatabase(c.String("db")) + library := newLibrary(newLibraryOptions{db}) + + for _, id := range c.Args().Slice() { + if err := library.delete(id); err != nil { + log.Fatal(err) + } + } + + return nil + }, + }, + &cli.Command{ + Name: "view", + Flags: flags, + Action: func(c *cli.Context) error { + db := newDatabase(c.String("db")) + library := newLibrary(newLibraryOptions{db}) + + for _, id := range c.Args().Slice() { + b, err := library.get(id) + if err != nil { + log.Fatal(err) + } + + json, err := json.Marshal(&b) + if err != nil { + log.Fatal(err) + } + + fmt.Println(string(json)) + } + + return nil + }, + }, + &cli.Command{ + Name: "download", + Flags: flags, + Action: func(c *cli.Context) error { + newDatabase(c.String("db")) + + return nil + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} diff --git a/migrations/0_init.sql b/migrations/0_init.sql new file mode 100644 index 0000000..040d179 --- /dev/null +++ b/migrations/0_init.sql @@ -0,0 +1,12 @@ +-- migrate:up +create table if not exists migrations ( + id integer primary key +); + +insert into schema_migrations(version) +select id +from migrations; + +drop table if exists migrations; + +-- migrate:down diff --git a/migrations/1_init.sql b/migrations/1_platforms_and_books.sql similarity index 93% rename from migrations/1_init.sql rename to migrations/1_platforms_and_books.sql index 7b2d1b5..40d3a2a 100644 --- a/migrations/1_init.sql +++ b/migrations/1_platforms_and_books.sql @@ -1,3 +1,4 @@ +-- migrate:up create table platforms ( id integer primary key autoincrement, name text unique not null, @@ -14,3 +15,5 @@ create table books ( insert into platforms(name) values ('dmm-books'), ('google-play-books'); + +-- migrate:down diff --git a/migrations/2_add_title_authors.sql b/migrations/2_add_title_authors.sql index 1f9001b..adbe541 100644 --- a/migrations/2_add_title_authors.sql +++ b/migrations/2_add_title_authors.sql @@ -1,2 +1,5 @@ +-- migrate:up alter table books add column title text not null default ''; alter table books add column authors json not null default '[]'; + +-- migrate:down diff --git a/migrations/3_add_fanza_doujin.sql b/migrations/3_add_fanza_doujin.sql index 890162b..4be0df2 100644 --- a/migrations/3_add_fanza_doujin.sql +++ b/migrations/3_add_fanza_doujin.sql @@ -1,2 +1,5 @@ +-- migrate:up insert into platforms(name) values ('fanza-doujin'); + +-- migrate:down diff --git a/migrations/4_add_dlsite_maniax.sql b/migrations/4_add_dlsite_maniax.sql index 3447cc3..8e45fb5 100644 --- a/migrations/4_add_dlsite_maniax.sql +++ b/migrations/4_add_dlsite_maniax.sql @@ -1,2 +1,5 @@ +-- migrate:up insert into platforms(name) values ('dlsite-maniax'); + +-- migrate:down