diff --git a/browser.go b/browser.go new file mode 100644 index 0000000..07a1610 --- /dev/null +++ b/browser.go @@ -0,0 +1,134 @@ +package main + +import ( + "encoding/base64" + "errors" + "net/http" + "net/url" + "strings" + + "github.com/playwright-community/playwright-go" +) + +type RawImageFile struct { + URL string + Blocks []map[string]int + Width *int + Height *int +} + +type File struct { + ContentType string + Data []byte +} + +var draw = ` +async function drawImage(imageFile) { + const canvas = Object.assign(document.createElement("canvas"), { + width: imageFile.width, + height: imageFile.height, + }); + const image = (await new Promise((resolve) => { + Object.assign(new Image(), { + crossOrigin: "use-credentials", + src: imageFile.url, + onload() { + resolve(this); + }, + }); + })); + const ctx = canvas.getContext("2d"); + for (const q of imageFile.blocks) { + ctx.drawImage(image, q.destX, q.destY, q.width, q.height, q.srcX, q.srcY, q.width, q.height); + } + const dataUrl = canvas.toDataURL(); + return dataUrl; +} +` + +var fetch = ` +async function fetchImage(imageFile) { + const res = await fetch(imageFile.url); + const blob = await res.blob(); + const dataUrl = await new Promise((resolve, reject) => { + const fileReader = Object.assign(new FileReader(), { + onload() { + resolve(this.result); + }, + onerror(e) { + const error = new Error(` + "`${e.type}: ${e.message}`" + `); + reject(error); + }, + }); + fileReader.readAsDataURL(blob); + }); + return dataUrl; +} +` + +func parseDataURL(dataURL string) (*File, error) { + if !strings.HasPrefix(dataURL, "data:") { + return nil, errors.New("not data scheme") + } + parts := strings.SplitN(dataURL, ",", 2) + if len(parts) != 2 { + return nil, errors.New("invalid data URL") + } + raw, err := url.PathUnescape(parts[1]) + if err != nil { + return nil, err + } + + var data []byte + + if strings.HasSuffix(parts[0], ";base64") { + data, err = base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, err + } + } else { + data = []byte(raw) + } + + file := File{ + ContentType: http.DetectContentType(data), + Data: data, + } + + return &file, nil +} + +type BrowserContext interface { + *playwright.BrowserContext +} + +// TODO +// export type Browser = { +// loadBrowserContext(platform: TPlatform): Promise; +// saveBrowserContext(platform: TPlatform, ctx: BrowserContext): Promise; +// newContext(): Promise; +// close(): Promise; +// drawImage( +// pageOrFrame: Playwright.Page | Playwright.Frame, +// imageFile: ImageFile, +// ): Promise; +// }; + +type Browser struct { +} + +func (b *Browser) New() (*Browser, error) { + pw, err := playwright.Run() + if err != nil { + return nil, err + } + browser, err := pw.Chromium.Launch() + if err != nil { + return nil, err + } + + b.Pw = pw + b.Browser = browser + + return b, nil +} diff --git a/database.go b/database.go new file mode 100644 index 0000000..00f4f74 --- /dev/null +++ b/database.go @@ -0,0 +1,49 @@ +package main + +import ( + "database/sql" + "embed" + "log" + "net/url" + "os" + "path/filepath" + + "github.com/amacneil/dbmate/v2/pkg/dbmate" + _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" + _ "github.com/mattn/go-sqlite3" +) + +//go:embed migrations/*.sql +var fs embed.FS + +type Database struct { + *sql.DB +} + +func NewDatabase(path string) *Database { + url, err := url.Parse("sqlite3://") + if err != nil { + log.Fatal(err) + } + url.Path, err = filepath.Abs(path) + if err != nil { + log.Fatal(err) + } + + dbmate := dbmate.New(url) + dbmate.AutoDumpSchema = false + dbmate.FS = fs + dbmate.MigrationsDir = []string{"migrations"} + dbmate.CreateAndMigrate() + + if err := os.Chmod(url.Path, 0600); err != nil { + log.Fatal(err) + } + + db, err := sql.Open("sqlite3", url.Path) + if err != nil { + log.Fatal(err) + } + + return &Database{db} +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6cbb720 --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module git.fogtype.com/nebel/gadl + +go 1.21.5 + +require ( + github.com/amacneil/dbmate/v2 v2.9.0 + github.com/mattn/go-sqlite3 v1.14.18 + github.com/playwright-community/playwright-go v0.3900.1 + github.com/urfave/cli/v2 v2.26.0 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + go.uber.org/multierr v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ef88a51 --- /dev/null +++ b/go.sum @@ -0,0 +1,49 @@ +github.com/amacneil/dbmate/v2 v2.9.0 h1:uXBlYKEQJL2gwXdiSlJLcXpgpibeak3OY0NN0oM/TCM= +github.com/amacneil/dbmate/v2 v2.9.0/go.mod h1:ygYXKjaOsIUnIZa/jfyosyfN2BXwwRY8uDGGTvQCADQ= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= +github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/playwright-community/playwright-go v0.3900.1 h1:8BkmDxVzLTp3USQ50EyXJSXcz0XDMwNP5y29lHIZ9Fc= +github.com/playwright-community/playwright-go v0.3900.1/go.mod h1:mbNzMqt04IVRdhVfXWqmCxd81gCdL3BA5hj6/pVAIqM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= +github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 h1:qXafrlZL1WsJW5OokjraLLRURHiw0OzKHD/RNdspp4w= +github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQxz6hGoNFBC4nIx+CxZhI3nne5RmIOlT/MXcSD4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..2af7867 --- /dev/null +++ b/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "log" + "os" + + _ "github.com/mattn/go-sqlite3" + "github.com/playwright-community/playwright-go" + "github.com/urfave/cli/v2" +) + +var version = "v1.0.0" + +/* +TODO + +- browser + - install +- library +- platform + +*/ + +func main() { + NewDatabase("gadl.db") + + flags := []cli.Flag{ + &cli.StringFlag{ + Name: "db", + Usage: "database file path", + Value: "gadl.db", + }, + } + + app := &cli.App{ + Name: "gadl", + Usage: "Manga Downloader", + Version: version, + Flags: flags, + Commands: cli.Commands{ + &cli.Command{ + Name: "download", + Flags: flags, + Action: func(ctx *cli.Context) error { + fmt.Println(ctx.String("db")) + + return nil + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} + +func _main() { + name := flag.String("name", "world", "name") + flag.Parse() + fmt.Printf("Hello, %s!\n", *name) + + db, err := sql.Open("sqlite3", "gadl.db") + if err != nil { + log.Fatal(err) + } + defer db.Close() + rows, err := db.Query("select id from books") + if err != nil { + log.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var id int + if err = rows.Scan(&id); err != nil { + log.Fatal(err) + } + fmt.Println(id) + } + + pw, err := playwright.Run() + if err != nil { + log.Fatal(err) + } + browser, err := pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{Headless: playwright.Bool(false)}) + if err != nil { + log.Fatal(err) + } + page, err := browser.NewPage() + if err != nil { + log.Fatal(err) + } + if _, err = page.Goto("https://news.ycombinator.com"); err != nil { + log.Fatal(err) + } + if err = browser.Close(); err != nil { + log.Fatal(err) + } + if err = pw.Stop(); err != nil { + log.Fatal(err) + } +} diff --git a/migrations/0_init.sql b/migrations/0_init.sql new file mode 100644 index 0000000..040d179 --- /dev/null +++ b/migrations/0_init.sql @@ -0,0 +1,12 @@ +-- migrate:up +create table if not exists migrations ( + id integer primary key +); + +insert into schema_migrations(version) +select id +from migrations; + +drop table if exists migrations; + +-- migrate:down diff --git a/migrations/1_init.sql b/migrations/1_platforms_and_books.sql similarity index 93% rename from migrations/1_init.sql rename to migrations/1_platforms_and_books.sql index 7b2d1b5..40d3a2a 100644 --- a/migrations/1_init.sql +++ b/migrations/1_platforms_and_books.sql @@ -1,3 +1,4 @@ +-- migrate:up create table platforms ( id integer primary key autoincrement, name text unique not null, @@ -14,3 +15,5 @@ create table books ( insert into platforms(name) values ('dmm-books'), ('google-play-books'); + +-- migrate:down diff --git a/migrations/2_add_title_authors.sql b/migrations/2_add_title_authors.sql index 1f9001b..adbe541 100644 --- a/migrations/2_add_title_authors.sql +++ b/migrations/2_add_title_authors.sql @@ -1,2 +1,5 @@ +-- migrate:up alter table books add column title text not null default ''; alter table books add column authors json not null default '[]'; + +-- migrate:down diff --git a/migrations/3_add_fanza_doujin.sql b/migrations/3_add_fanza_doujin.sql index 890162b..4be0df2 100644 --- a/migrations/3_add_fanza_doujin.sql +++ b/migrations/3_add_fanza_doujin.sql @@ -1,2 +1,5 @@ +-- migrate:up insert into platforms(name) values ('fanza-doujin'); + +-- migrate:down diff --git a/migrations/4_add_dlsite_maniax.sql b/migrations/4_add_dlsite_maniax.sql index 3447cc3..8e45fb5 100644 --- a/migrations/4_add_dlsite_maniax.sql +++ b/migrations/4_add_dlsite_maniax.sql @@ -1,2 +1,5 @@ +-- migrate:up insert into platforms(name) values ('dlsite-maniax'); + +-- migrate:down