diff --git a/README.md b/README.md index e239297..e311aa8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ $ npx https://git.fogtype.com/nebel/gadl/archive/main.tar.gz --help ## Supported Sites +- DLsite 同人 - DMM ブックス (漫画) - FANZA 同人 - Google Play ブックス (漫画) diff --git a/browser.ts b/browser.ts index 883b1ac..3124529 100644 --- a/browser.ts +++ b/browser.ts @@ -66,6 +66,11 @@ async function fetchImage(imageFile: ImageFile): Promise { return dataUrl; } +async function dataUrlToBlob(dataUrl: string): Promise { + const res = await fetch(dataUrl); + return await res.blob(); +} + export type Browser = { loadBrowserContext(platform: TPlatform): Promise; saveBrowserContext(platform: TPlatform, ctx: BrowserContext): Promise; @@ -74,7 +79,7 @@ export type Browser = { drawImage( pageOrFrame: Playwright.Page | Playwright.Frame, imageFile: ImageFile, - ): Promise; + ): Promise; }; export type BrowserContext = Playwright.BrowserContext; @@ -124,22 +129,38 @@ export async function createBrowser({ async drawImage( pageOrFrame: Playwright.Page | Playwright.Frame, imageFile: ImageFile, - ): Promise { + ): Promise { if (Array.isArray(imageFile.blocks) && imageFile.blocks.length > 0) { - return await pageOrFrame.evaluate(drawImage, imageFile); + const dataUrl = await pageOrFrame.evaluate(drawImage, imageFile); + return await dataUrlToBlob(dataUrl); } if (imageFile.url.startsWith("blob:")) { - return await pageOrFrame.evaluate(fetchImage, imageFile); + const dataUrl = await pageOrFrame.evaluate(fetchImage, imageFile); + return await dataUrlToBlob(dataUrl); } const page = "page" in pageOrFrame ? pageOrFrame.page() : pageOrFrame; const res = await page.context().request.get(imageFile.url); + const headers = res.headers(); const buffer = await res.body(); - const type = res.headers()["content-type"]; - const base64 = buffer.toString("base64"); - return `data:${type};base64,${base64}`; + let type = headers["content-type"]; + + if (type === "binary/octet-stream") { + const [, extension] = + /^attachment *; *filename="[^"]+[.]([^.]+)" *(?:$|;)/i.exec( + headers["content-disposition"], + ) ?? []; + + switch (extension) { + case "zip": + type = "application/zip"; + break; + } + } + + return new Blob([buffer], { type }); }, }; } diff --git a/main.ts b/main.ts index a053dd5..6340013 100644 --- a/main.ts +++ b/main.ts @@ -82,7 +82,7 @@ const options = { async run() { const db = await createDatabase(args.values.db!); const library = createLibrary(db); - const book = await library.get(Number(args.values.view!)); + const book = await library.get(args.values.view!); if (!book) { process.exit(1); diff --git a/migrations/4_add_dlsite_maniax.sql b/migrations/4_add_dlsite_maniax.sql new file mode 100644 index 0000000..3447cc3 --- /dev/null +++ b/migrations/4_add_dlsite_maniax.sql @@ -0,0 +1,2 @@ +insert into platforms(name) values + ('dlsite-maniax'); diff --git a/package-lock.json b/package-lock.json index 7984233..b3e588b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fogtype/gadl", - "version": "1.2.0", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@fogtype/gadl", - "version": "1.2.0", + "version": "1.3.0", "license": "AGPL-3.0", "dependencies": { "fflate": "^0.8.1", diff --git a/package.json b/package.json index 7ab4522..83f58f9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fogtype/gadl", - "version": "1.2.0", + "version": "1.3.0", "license": "AGPL-3.0", "type": "module", "bin": "bin/run.js", diff --git a/platform.ts b/platform.ts index 6e62715..0210ed4 100644 --- a/platform.ts +++ b/platform.ts @@ -3,11 +3,13 @@ import path from "node:path"; import type { Book } from "./library"; import type { Browser } from "./browser"; import type { Database } from "./database"; +import { DlsiteManiax } from "./platforms/dlsite-maniax"; import { DmmBooks } from "./platforms/dmm-books"; import { FanzaDoujin } from "./platforms/fanza-doujin"; import { GooglePlayBooks } from "./platforms/google-play-books"; const platforms = { + "dlsite-maniax": DlsiteManiax, "dmm-books": DmmBooks, "fanza-doujin": FanzaDoujin, "google-play-books": GooglePlayBooks, @@ -45,7 +47,7 @@ export function createPlatform(opts: { await fs.mkdir(path.dirname(dir), { recursive: true }); await fs.mkdir(dir); - const files: Array<() => Promise> = await platform.getFiles(book); + const files: Array<() => Promise> = await platform.getFiles(book); const digits = String(files.length).length; function pad(n: string) { @@ -59,32 +61,48 @@ export function createPlatform(opts: { "application/vnd.comicbook+zip": "cbz", }; - for (const [n, dataUrl] of Object.entries(files)) { - const [prefix, base64] = (await dataUrl()).split(",", 2); - const [, type, encoding] = - /^data:([^;]*)(;base64)?$/.exec(prefix) ?? []; - - const extension = supportedTypes[type]; + for (const [n, getBlob] of Object.entries(files)) { + const blob = await getBlob(); + const extension = supportedTypes[blob.type]; if (!extension) { throw new Error( - `It was ${type}. The image must be a file of type: ${[ + `It was ${blob.type}. The image must be a file of type: ${[ ...Object.keys(supportedTypes), ].join(", ")}.`, ); } - if (encoding !== ";base64") { - throw new Error("Only base64 is supported."); - } - - const buffer = Buffer.from(base64, "base64"); + const buffer = Buffer.from(await blob.arrayBuffer()); await fs.writeFile(`${dir}/${pad(n)}.${extension}`, buffer); } process.stderr.write(`\n`); }, + async login() { + const ctx = await opts.browser.newContext(); + const page = await ctx.newPage(); + + for (const loginEndpoint of platform.loginEndpoints) { + await page.goto(loginEndpoint); + await page.waitForURL(platform.loginSuccessUrl, { timeout: 0 }); + } + + await opts.browser.saveBrowserContext(opts.platform, ctx); + }, + async logout() { + try { + const ctx = await opts.browser.loadBrowserContext(opts.platform); + const page = await ctx.newPage(); + + for (const logoutEndpoint of platform.logoutEndpoints) { + await page.goto(logoutEndpoint); + } + } catch (error) { + process.stderr.write(`Warning: ${(error as Error).message}\n`); + } + await opts.db.run( `update platforms set secrets = 'null' where name = ?`, opts.platform, diff --git a/platforms/dlsite-maniax.ts b/platforms/dlsite-maniax.ts new file mode 100644 index 0000000..1e7bd9b --- /dev/null +++ b/platforms/dlsite-maniax.ts @@ -0,0 +1,104 @@ +import type { Book } from "../library"; +import type { Browser, BrowserContext, ImageFile } from "../browser"; + +export function DlsiteManiax(browser: Browser) { + async function* getAllBooks(ctx: BrowserContext): AsyncGenerator { + const totalCountEndpoint = "https://play.dlsite.com/api/product_count"; + const endpoint = "https://play.dlsite.com/api/purchases"; + const pager = { + page: 1, + perPage: 50, + totalCount: Infinity, + }; + + const res = await ctx.request.get(totalCountEndpoint); + const body: { + user: number; + } = await res.json(); + + pager.totalCount = body.user; + + while ((pager.page - 1) * pager.perPage <= pager.totalCount) { + const res = await ctx.request.get(`${endpoint}?page=${pager.page}`); + + if (!res.ok()) { + throw new Error(`${res.status()} ${res.statusText()}`); + } + + const body: { + limit: number; + works: Array<{ + workno: number; + name: { + ja_JP: string; + }; + maker: { + name: { + ja_JP: string; + }; + }; + }>; + } = await res.json(); + + for (const work of Object.values(body.works).flat()) { + yield { + id: NaN, + platform: "dlsite-maniax", + readerUrl: `https://play.dlsite.com/#/work/${work.workno}`, + title: work.name.ja_JP || "", + authors: [work.maker.name.ja_JP || ""], + }; + + process.stderr.write("."); + } + + pager.page += 1; + pager.perPage = body.limit; + } + } + + return { + async *pull(): AsyncGenerator { + const ctx = await browser.loadBrowserContext("dlsite-maniax"); + + yield* getAllBooks(ctx); + + process.stderr.write(`\n`); + }, + + async getFiles(book: Book): Promise Promise>> { + const ctx = await browser.loadBrowserContext("dlsite-maniax"); + const page = await ctx.newPage(); + + await page.goto(book.readerUrl); + + const [, workId] = + /^https:[/][/]play[.]dlsite[.]com[/]#[/]work[/]([^/]+)/.exec( + book.readerUrl, + ) ?? []; + + if (!workId) { + throw new Error(`workId is not included: ${book.readerUrl}`); + } + + const url = `https://www.dlsite.com/home/download/=/product_id/${workId}.html`; + const imageFile = { url }; + + return [ + async () => { + const blob = await browser.drawImage(page, imageFile); + + process.stderr.write("."); + + return blob; + }, + ]; + }, + loginEndpoints: ["https://www.dlsite.com/home/login"], + loginSuccessUrl: (url: URL) => url.origin === "https://www.dlsite.com", + logoutEndpoints: ["https://www.dlsite.com/home/logout"], + }; +} + +DlsiteManiax.siteUrl = (url: URL) => + url.href.startsWith("https://play.dlsite.com/#/work/"); diff --git a/platforms/dmm-books.ts b/platforms/dmm-books.ts index 07b261f..2cd3f1c 100644 --- a/platforms/dmm-books.ts +++ b/platforms/dmm-books.ts @@ -89,7 +89,7 @@ export function DmmBooks(browser: Browser) { totalCount: Infinity, }; - while (pager.page * pager.perPage <= pager.totalCount) { + while ((pager.page - 1) * pager.perPage <= pager.totalCount) { const res = await ctx.request.get(`${endpoint}&page=${pager.page}`); if (!res.ok()) { @@ -136,7 +136,7 @@ export function DmmBooks(browser: Browser) { totalCount: Infinity, }; - while (pager.page * pager.perPage <= pager.totalCount) { + while ((pager.page - 1) * pager.perPage <= pager.totalCount) { const res = await ctx.request.get(`${endpoint}&page=${pager.page}`); if (!res.ok()) { @@ -173,19 +173,6 @@ export function DmmBooks(browser: Browser) { } return { - async login() { - const ctx = await browser.newContext(); - const page = await ctx.newPage(); - await page.goto("https://accounts.dmm.com/service/login/password"); - await page.waitForURL("https://www.dmm.com/", { timeout: 0 }); - await page.goto("https://www.dmm.com/service/-/exchange"); - await Promise.race([ - page.waitForURL("https://www.dmm.com/", { timeout: 0 }), - page.waitForURL("https://www.dmm.co.jp/top/", { timeout: 0 }), - ]); - await browser.saveBrowserContext("dmm-books", ctx); - }, - async *pull(): AsyncGenerator { const ctx = await browser.loadBrowserContext("dmm-books"); @@ -194,7 +181,7 @@ export function DmmBooks(browser: Browser) { process.stderr.write(`\n`); }, - async getFiles(book: Book): Promise Promise>> { + async getFiles(book: Book): Promise Promise>> { const ctx = await browser.loadBrowserContext("dmm-books"); const page = await ctx.newPage(); @@ -203,13 +190,23 @@ export function DmmBooks(browser: Browser) { const imageFiles = await page.evaluate(getImageFiles); return imageFiles.map((imageFile) => async () => { - const dataUrl = await browser.drawImage(page, imageFile); + const blob = await browser.drawImage(page, imageFile); process.stderr.write("."); - return dataUrl; + return blob; }); }, + loginEndpoints: [ + "https://accounts.dmm.com/service/login/password", + "https://www.dmm.com/service/-/exchange", + ], + loginSuccessUrl: (url: URL) => + ["https://www.dmm.com/", "https://www.dmm.co.jp/top/"].includes(url.href), + logoutEndpoints: [ + "https://accounts.dmm.com/service/logout", + "https://accounts.dmm.co.jp/service/logout", + ], }; } diff --git a/platforms/fanza-doujin.ts b/platforms/fanza-doujin.ts index 7111296..b3a633f 100644 --- a/platforms/fanza-doujin.ts +++ b/platforms/fanza-doujin.ts @@ -12,7 +12,7 @@ export function FanzaDoujin(browser: Browser) { totalCount: Infinity, }; - while (pager.page * pager.perPage <= pager.totalCount) { + while ((pager.page - 1) * pager.perPage <= pager.totalCount) { const res = await ctx.request.get(`${endpoint}&page=${pager.page}`); if (!res.ok()) { @@ -51,14 +51,6 @@ export function FanzaDoujin(browser: Browser) { } return { - async login() { - const ctx = await browser.newContext(); - const page = await ctx.newPage(); - await page.goto("https://accounts.dmm.co.jp/service/login/password"); - await page.waitForURL("https://www.dmm.co.jp/top/", { timeout: 0 }); - await browser.saveBrowserContext("fanza-doujin", ctx); - }, - async *pull(): AsyncGenerator { const ctx = await browser.loadBrowserContext("fanza-doujin"); @@ -67,7 +59,7 @@ export function FanzaDoujin(browser: Browser) { process.stderr.write(`\n`); }, - async getFiles(book: Book): Promise Promise>> { + async getFiles(book: Book): Promise Promise>> { const ctx = await browser.loadBrowserContext("fanza-doujin"); const page = await ctx.newPage(); @@ -117,13 +109,16 @@ export function FanzaDoujin(browser: Browser) { } return imageFiles.map((imageFile) => async () => { - const dataUrl = await browser.drawImage(page, imageFile); + const blob = await browser.drawImage(page, imageFile); process.stderr.write("."); - return dataUrl; + return blob; }); }, + loginEndpoints: ["https://accounts.dmm.co.jp/service/login/password"], + loginSuccessUrl: (url: URL) => url.href === "https://www.dmm.co.jp/top/", + logoutEndpoints: ["https://accounts.dmm.co.jp/service/logout"], }; } diff --git a/platforms/google-play-books.ts b/platforms/google-play-books.ts index 323312f..0be87db 100644 --- a/platforms/google-play-books.ts +++ b/platforms/google-play-books.ts @@ -39,17 +39,6 @@ async function getImageFiles(): Promise> { export function GooglePlayBooks(browser: Browser) { return { - async login() { - const ctx = await browser.newContext(); - const page = await ctx.newPage(); - await page.goto("https://accounts.google.com"); - await page.waitForURL( - (url) => url.origin === "https://myaccount.google.com", - { timeout: 0 }, - ); - await browser.saveBrowserContext("google-play-books", ctx); - }, - async *pull(): AsyncGenerator { const ctx = await browser.loadBrowserContext("google-play-books"); const page = await ctx.newPage(); @@ -76,7 +65,7 @@ export function GooglePlayBooks(browser: Browser) { process.stderr.write(`\n`); }, - async getFiles(book: Book): Promise Promise>> { + async getFiles(book: Book): Promise Promise>> { const ctx = await browser.loadBrowserContext("google-play-books"); const page = await ctx.newPage(); @@ -118,7 +107,7 @@ export function GooglePlayBooks(browser: Browser) { }); } - const fileMap: Map Promise> = new Map(); + const fileMap: Map Promise> = new Map(); while (await next()) { const imageFiles = await frame.evaluate(getImageFiles); @@ -126,16 +115,20 @@ export function GooglePlayBooks(browser: Browser) { for (const imageFile of imageFiles) { if (fileMap.has(imageFile.url)) continue; - const dataUrl = await browser.drawImage(frame, imageFile); + const blob = await browser.drawImage(frame, imageFile); process.stderr.write("."); - fileMap.set(imageFile.url, async () => dataUrl); + fileMap.set(imageFile.url, async () => blob); } } return [...fileMap.values()]; }, + loginEndpoints: ["https://accounts.google.com"], + loginSuccessUrl: (url: URL) => + url.origin === "https://myaccount.google.com", + logoutEndpoints: ["https://accounts.google.com/Logout"], }; }