nebel/gadl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
Nebel 2023-12-13 23:28:05 +09:00
parent 59c28ca0cd
commit de6b53e975
Signed by: nebel
GPG key ID: 79807D08C6EF6460
10 changed files with 379 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
browser.go Normal file
View file

@ -0,0 +1,131 @@
package main
import (
"encoding/base64"
"errors"
"net/http"
"net/url"
"strings"
"github.com/playwright-community/playwright-go"
)
type RawImageFile struct {
URL string
Blocks []map[string]int
Width *int
Height *int
}
type File struct {
ContentType string
Data []byte
}
var draw = `
async function drawImage(imageFile) {
const canvas = Object.assign(document.createElement("canvas"), {
width: imageFile.width,
height: imageFile.height,
});
const image = (await new Promise((resolve) => {
Object.assign(new Image(), {
crossOrigin: "use-credentials",
src: imageFile.url,
onload() {
resolve(this);
},
});
}));
const ctx = canvas.getContext("2d");
for (const q of imageFile.blocks) {
ctx.drawImage(image, q.destX, q.destY, q.width, q.height, q.srcX, q.srcY, q.width, q.height);
}
const dataUrl = canvas.toDataURL();
return dataUrl;
}
`
var fetch = `
async function fetchImage(imageFile) {
const res = await fetch(imageFile.url);
const blob = await res.blob();
const dataUrl = await new Promise((resolve, reject) => {
const fileReader = Object.assign(new FileReader(), {
onload() {
resolve(this.result);
},
onerror(e) {
const error = new Error(` + "`${e.type}: ${e.message}`" + `);
reject(error);
},
});
fileReader.readAsDataURL(blob);
});
return dataUrl;
}
`
func parseDataURL(dataURL string) (*File, error) {
if !strings.HasPrefix(dataURL, "data:") {
return nil, errors.New("not data scheme")
}
parts := strings.SplitN(dataURL, ",", 2)
if len(parts) != 2 {
return nil, errors.New("invalid data URL")
}
raw, err := url.PathUnescape(parts[1])
if err != nil {
return nil, err
}
var data []byte
if strings.HasSuffix(parts[0], ";base64") {
data, err = base64.StdEncoding.DecodeString(raw)
if err != nil {
return nil, err
}
} else {
data = []byte(raw)
}
file := File{
ContentType: http.DetectContentType(data),
Data: data,
}
return &file, nil
}
type BrowserContext interface {
*playwright.BrowserContext
}
type Browser struct {
browser playwright.Browser
}
// TODO
// Browser loadBrowserContext(platform: TPlatform): Promise<Playwright.BrowserContext>;
// Browser saveBrowserContext(platform: TPlatform, ctx: BrowserContext): Promise<void>;
// Browser newContext(): Promise<Playwright.BrowserContext>;
// Browser close(): Promise<void>;
// Browser drawImage(
// pageOrFrame: Playwright.Page | Playwright.Frame,
// imageFile: ImageFile,
// ): Promise<Blob>;
// install
// newじゃないか? run/stop? launch/close?
func New() (*Browser, error) {
pw, err := playwright.Run()
if err != nil {
return nil, err
}
browser, err := pw.Chromium.Launch()
if err != nil {
return nil, err
}
return &Browser{browser}, nil
}

49
database.go Normal file
View file

@ -0,0 +1,49 @@
package main
import (
"database/sql"
"embed"
"log"
"net/url"
"os"
"path/filepath"
"github.com/amacneil/dbmate/v2/pkg/dbmate"
_ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite"
_ "github.com/mattn/go-sqlite3"
)
//go:embed migrations/*.sql
var fs embed.FS
type Database struct {
*sql.DB
}
func NewDatabase(path string) *Database {
url, err := url.Parse("sqlite3://")
if err != nil {
log.Fatal(err)
}
url.Path, err = filepath.Abs(path)
if err != nil {
log.Fatal(err)
}
dbmate := dbmate.New(url)
dbmate.AutoDumpSchema = false
dbmate.FS = fs
dbmate.MigrationsDir = []string{"migrations"}
dbmate.CreateAndMigrate()
if err := os.Chmod(url.Path, 0600); err != nil {
log.Fatal(err)
}
db, err := sql.Open("sqlite3", url.Path)
if err != nil {
log.Fatal(err)
}
return &Database{db}
}

21
go.mod Normal file
View file

@ -0,0 +1,21 @@
module git.fogtype.com/nebel/gadl
go 1.21.5
require (
github.com/amacneil/dbmate/v2 v2.9.0
github.com/mattn/go-sqlite3 v1.14.18
github.com/playwright-community/playwright-go v0.3900.1
github.com/urfave/cli/v2 v2.26.0
)
require (
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/multierr v1.11.0 // indirect
)

49
go.sum Normal file
View file

@ -0,0 +1,49 @@
github.com/amacneil/dbmate/v2 v2.9.0 h1:uXBlYKEQJL2gwXdiSlJLcXpgpibeak3OY0NN0oM/TCM=
github.com/amacneil/dbmate/v2 v2.9.0/go.mod h1:ygYXKjaOsIUnIZa/jfyosyfN2BXwwRY8uDGGTvQCADQ=
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/playwright-community/playwright-go v0.3900.1 h1:8BkmDxVzLTp3USQ50EyXJSXcz0XDMwNP5y29lHIZ9Fc=
github.com/playwright-community/playwright-go v0.3900.1/go.mod h1:mbNzMqt04IVRdhVfXWqmCxd81gCdL3BA5hj6/pVAIqM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 h1:qXafrlZL1WsJW5OokjraLLRURHiw0OzKHD/RNdspp4w=
github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQxz6hGoNFBC4nIx+CxZhI3nne5RmIOlT/MXcSD4=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

105
main.go Normal file
View file

@ -0,0 +1,105 @@
package main
import (
"database/sql"
"flag"
"fmt"
"log"
"os"
_ "github.com/mattn/go-sqlite3"
"github.com/playwright-community/playwright-go"
"github.com/urfave/cli/v2"
)
var version = "v1.0.0"
/*
TODO
- browser
- install
- library
- platform
*/
func main() {
NewDatabase("gadl.db")
flags := []cli.Flag{
&cli.StringFlag{
Name: "db",
Usage: "database file path",
Value: "gadl.db",
},
}
app := &cli.App{
Name: "gadl",
Usage: "Manga Downloader",
Version: version,
Flags: flags,
Commands: cli.Commands{
&cli.Command{
Name: "download",
Flags: flags,
Action: func(ctx *cli.Context) error {
fmt.Println(ctx.String("db"))
return nil
},
},
},
}
if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}
func _main() {
name := flag.String("name", "world", "name")
flag.Parse()
fmt.Printf("Hello, %s!\n", *name)
db, err := sql.Open("sqlite3", "gadl.db")
if err != nil {
log.Fatal(err)
}
defer db.Close()
rows, err := db.Query("select id from books")
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var id int
if err = rows.Scan(&id); err != nil {
log.Fatal(err)
}
fmt.Println(id)
}
pw, err := playwright.Run()
if err != nil {
log.Fatal(err)
}
browser, err := pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{Headless: playwright.Bool(false)})
if err != nil {
log.Fatal(err)
}
page, err := browser.NewPage()
if err != nil {
log.Fatal(err)
}
if _, err = page.Goto("https://news.ycombinator.com"); err != nil {
log.Fatal(err)
}
if err = browser.Close(); err != nil {
log.Fatal(err)
}
if err = pw.Stop(); err != nil {
log.Fatal(err)
}
}

12
migrations/0_init.sql Normal file
View file

@ -0,0 +1,12 @@
-- migrate:up
create table if not exists migrations (
id integer primary key
);
insert into schema_migrations(version)
select id
from migrations;
drop table if exists migrations;
-- migrate:down

3
migrations/1_init.sql → migrations/1_platforms_and_books.sql
View file

@ -1,3 +1,4 @@
-- migrate:up
create table platforms (
id integer primary key autoincrement,
name text unique not null,
@ -14,3 +15,5 @@ create table books (
insert into platforms(name) values
('dmm-books'),
('google-play-books');
-- migrate:down

3
migrations/2_add_title_authors.sql
View file

@ -1,2 +1,5 @@
-- migrate:up
alter table books add column title text not null default '';
alter table books add column authors json not null default '[]';
-- migrate:down

3
migrations/3_add_fanza_doujin.sql
View file

@ -1,2 +1,5 @@
-- migrate:up
insert into platforms(name) values
('fanza-doujin');
-- migrate:down

3
migrations/4_add_dlsite_maniax.sql
View file

@ -1,2 +1,5 @@
-- migrate:up
insert into platforms(name) values
('dlsite-maniax');
-- migrate:down