dlsite-maniax
This commit is contained in:
parent
7547a34e03
commit
a53de8e5dd
GPG key ID:
79807D08C6EF6460
11 changed files with 200 additions and 69 deletions
|
|
@ -9,6 +9,7 @@ $ npx https://git.fogtype.com/nebel/gadl/archive/main.tar.gz --help
|
|||
|
||||
## Supported Sites
|
||||
|
||||
- DLsite 同人
|
||||
- DMM ブックス (漫画)
|
||||
- FANZA 同人
|
||||
- Google Play ブックス (漫画)
|
||||
|
|
|
|||
35
browser.ts
35
browser.ts
|
|
@ -66,6 +66,11 @@ async function fetchImage(imageFile: ImageFile): Promise<string> {
|
|||
return dataUrl;
|
||||
}
|
||||
|
||||
async function dataUrlToBlob(dataUrl: string): Promise<Blob> {
|
||||
const res = await fetch(dataUrl);
|
||||
return await res.blob();
|
||||
}
|
||||
|
||||
export type Browser = {
|
||||
loadBrowserContext(platform: TPlatform): Promise<Playwright.BrowserContext>;
|
||||
saveBrowserContext(platform: TPlatform, ctx: BrowserContext): Promise<void>;
|
||||
|
|
@ -74,7 +79,7 @@ export type Browser = {
|
|||
drawImage(
|
||||
pageOrFrame: Playwright.Page | Playwright.Frame,
|
||||
imageFile: ImageFile,
|
||||
): Promise<string>;
|
||||
): Promise<Blob>;
|
||||
};
|
||||
|
||||
export type BrowserContext = Playwright.BrowserContext;
|
||||
|
|
@ -124,22 +129,38 @@ export async function createBrowser({
|
|||
async drawImage(
|
||||
pageOrFrame: Playwright.Page | Playwright.Frame,
|
||||
imageFile: ImageFile,
|
||||
): Promise<string> {
|
||||
): Promise<Blob> {
|
||||
if (Array.isArray(imageFile.blocks) && imageFile.blocks.length > 0) {
|
||||
return await pageOrFrame.evaluate(drawImage, imageFile);
|
||||
const dataUrl = await pageOrFrame.evaluate(drawImage, imageFile);
|
||||
return await dataUrlToBlob(dataUrl);
|
||||
}
|
||||
|
||||
if (imageFile.url.startsWith("blob:")) {
|
||||
return await pageOrFrame.evaluate(fetchImage, imageFile);
|
||||
const dataUrl = await pageOrFrame.evaluate(fetchImage, imageFile);
|
||||
return await dataUrlToBlob(dataUrl);
|
||||
}
|
||||
|
||||
const page = "page" in pageOrFrame ? pageOrFrame.page() : pageOrFrame;
|
||||
const res = await page.context().request.get(imageFile.url);
|
||||
const headers = res.headers();
|
||||
const buffer = await res.body();
|
||||
const type = res.headers()["content-type"];
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
return `data:${type};base64,${base64}`;
|
||||
let type = headers["content-type"];
|
||||
|
||||
if (type === "binary/octet-stream") {
|
||||
const [, extension] =
|
||||
/^attachment *; *filename="[^"]+[.]([^.]+)" *(?:$|;)/i.exec(
|
||||
headers["content-disposition"],
|
||||
) ?? [];
|
||||
|
||||
switch (extension) {
|
||||
case "zip":
|
||||
type = "application/zip";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new Blob([buffer], { type });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
2
main.ts
2
main.ts
|
|
@ -82,7 +82,7 @@ const options = {
|
|||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
const book = await library.get(Number(args.values.view!));
|
||||
const book = await library.get(args.values.view!);
|
||||
|
||||
if (!book) {
|
||||
process.exit(1);
|
||||
|
|
|
|||
2
migrations/4_add_dlsite_maniax.sql
Normal file
2
migrations/4_add_dlsite_maniax.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
insert into platforms(name) values
|
||||
('dlsite-maniax');
|
||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "@fogtype/gadl",
|
||||
"version": "1.2.0",
|
||||
"version": "1.3.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@fogtype/gadl",
|
||||
"version": "1.2.0",
|
||||
"version": "1.3.0",
|
||||
"license": "AGPL-3.0",
|
||||
"dependencies": {
|
||||
"fflate": "^0.8.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@fogtype/gadl",
|
||||
"version": "1.2.0",
|
||||
"version": "1.3.0",
|
||||
"license": "AGPL-3.0",
|
||||
"type": "module",
|
||||
"bin": "bin/run.js",
|
||||
|
|
|
|||
44
platform.ts
44
platform.ts
|
|
@ -3,11 +3,13 @@ import path from "node:path";
|
|||
import type { Book } from "./library";
|
||||
import type { Browser } from "./browser";
|
||||
import type { Database } from "./database";
|
||||
import { DlsiteManiax } from "./platforms/dlsite-maniax";
|
||||
import { DmmBooks } from "./platforms/dmm-books";
|
||||
import { FanzaDoujin } from "./platforms/fanza-doujin";
|
||||
import { GooglePlayBooks } from "./platforms/google-play-books";
|
||||
|
||||
const platforms = {
|
||||
"dlsite-maniax": DlsiteManiax,
|
||||
"dmm-books": DmmBooks,
|
||||
"fanza-doujin": FanzaDoujin,
|
||||
"google-play-books": GooglePlayBooks,
|
||||
|
|
@ -45,7 +47,7 @@ export function createPlatform(opts: {
|
|||
await fs.mkdir(path.dirname(dir), { recursive: true });
|
||||
await fs.mkdir(dir);
|
||||
|
||||
const files: Array<() => Promise<string>> = await platform.getFiles(book);
|
||||
const files: Array<() => Promise<Blob>> = await platform.getFiles(book);
|
||||
const digits = String(files.length).length;
|
||||
|
||||
function pad(n: string) {
|
||||
|
|
@ -59,32 +61,48 @@ export function createPlatform(opts: {
|
|||
"application/vnd.comicbook+zip": "cbz",
|
||||
};
|
||||
|
||||
for (const [n, dataUrl] of Object.entries(files)) {
|
||||
const [prefix, base64] = (await dataUrl()).split(",", 2);
|
||||
const [, type, encoding] =
|
||||
/^data:([^;]*)(;base64)?$/.exec(prefix) ?? [];
|
||||
|
||||
const extension = supportedTypes[type];
|
||||
for (const [n, getBlob] of Object.entries(files)) {
|
||||
const blob = await getBlob();
|
||||
const extension = supportedTypes[blob.type];
|
||||
if (!extension) {
|
||||
throw new Error(
|
||||
`It was ${type}. The image must be a file of type: ${[
|
||||
`It was ${blob.type}. The image must be a file of type: ${[
|
||||
...Object.keys(supportedTypes),
|
||||
].join(", ")}.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (encoding !== ";base64") {
|
||||
throw new Error("Only base64 is supported.");
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(base64, "base64");
|
||||
const buffer = Buffer.from(await blob.arrayBuffer());
|
||||
await fs.writeFile(`${dir}/${pad(n)}.${extension}`, buffer);
|
||||
}
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async login() {
|
||||
const ctx = await opts.browser.newContext();
|
||||
const page = await ctx.newPage();
|
||||
|
||||
for (const loginEndpoint of platform.loginEndpoints) {
|
||||
await page.goto(loginEndpoint);
|
||||
await page.waitForURL(platform.loginSuccessUrl, { timeout: 0 });
|
||||
}
|
||||
|
||||
await opts.browser.saveBrowserContext(opts.platform, ctx);
|
||||
},
|
||||
|
||||
async logout() {
|
||||
try {
|
||||
const ctx = await opts.browser.loadBrowserContext(opts.platform);
|
||||
const page = await ctx.newPage();
|
||||
|
||||
for (const logoutEndpoint of platform.logoutEndpoints) {
|
||||
await page.goto(logoutEndpoint);
|
||||
}
|
||||
} catch (error) {
|
||||
process.stderr.write(`Warning: ${(error as Error).message}\n`);
|
||||
}
|
||||
|
||||
await opts.db.run(
|
||||
`update platforms set secrets = 'null' where name = ?`,
|
||||
opts.platform,
|
||||
|
|
|
|||
104
platforms/dlsite-maniax.ts
Normal file
104
platforms/dlsite-maniax.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import type { Book } from "../library";
|
||||
import type { Browser, BrowserContext, ImageFile } from "../browser";
|
||||
|
||||
export function DlsiteManiax(browser: Browser) {
|
||||
async function* getAllBooks(ctx: BrowserContext): AsyncGenerator<Book> {
|
||||
const totalCountEndpoint = "https://play.dlsite.com/api/product_count";
|
||||
const endpoint = "https://play.dlsite.com/api/purchases";
|
||||
const pager = {
|
||||
page: 1,
|
||||
perPage: 50,
|
||||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
const res = await ctx.request.get(totalCountEndpoint);
|
||||
const body: {
|
||||
user: number;
|
||||
} = await res.json();
|
||||
|
||||
pager.totalCount = body.user;
|
||||
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}?page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const body: {
|
||||
limit: number;
|
||||
works: Array<{
|
||||
workno: number;
|
||||
name: {
|
||||
ja_JP: string;
|
||||
};
|
||||
maker: {
|
||||
name: {
|
||||
ja_JP: string;
|
||||
};
|
||||
};
|
||||
}>;
|
||||
} = await res.json();
|
||||
|
||||
for (const work of Object.values(body.works).flat()) {
|
||||
yield {
|
||||
id: NaN,
|
||||
platform: "dlsite-maniax",
|
||||
readerUrl: `https://play.dlsite.com/#/work/${work.workno}`,
|
||||
title: work.name.ja_JP || "",
|
||||
authors: [work.maker.name.ja_JP || ""],
|
||||
};
|
||||
|
||||
process.stderr.write(".");
|
||||
}
|
||||
|
||||
pager.page += 1;
|
||||
pager.perPage = body.limit;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("dlsite-maniax");
|
||||
|
||||
yield* getAllBooks(ctx);
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("dlsite-maniax");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
await page.goto(book.readerUrl);
|
||||
|
||||
const [, workId] =
|
||||
/^https:[/][/]play[.]dlsite[.]com[/]#[/]work[/]([^/]+)/.exec(
|
||||
book.readerUrl,
|
||||
) ?? [];
|
||||
|
||||
if (!workId) {
|
||||
throw new Error(`workId is not included: ${book.readerUrl}`);
|
||||
}
|
||||
|
||||
const url = `https://www.dlsite.com/home/download/=/product_id/${workId}.html`;
|
||||
const imageFile = { url };
|
||||
|
||||
return [
|
||||
async () => {
|
||||
const blob = await browser.drawImage(page, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
return blob;
|
||||
},
|
||||
];
|
||||
},
|
||||
loginEndpoints: ["https://www.dlsite.com/home/login"],
|
||||
loginSuccessUrl: (url: URL) => url.origin === "https://www.dlsite.com",
|
||||
logoutEndpoints: ["https://www.dlsite.com/home/logout"],
|
||||
};
|
||||
}
|
||||
|
||||
DlsiteManiax.siteUrl = (url: URL) =>
|
||||
url.href.startsWith("https://play.dlsite.com/#/work/");
|
||||
|
|
@ -89,7 +89,7 @@ export function DmmBooks(browser: Browser) {
|
|||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
while (pager.page * pager.perPage <= pager.totalCount) {
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
|
|
@ -136,7 +136,7 @@ export function DmmBooks(browser: Browser) {
|
|||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
while (pager.page * pager.perPage <= pager.totalCount) {
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
|
|
@ -173,19 +173,6 @@ export function DmmBooks(browser: Browser) {
|
|||
}
|
||||
|
||||
return {
|
||||
async login() {
|
||||
const ctx = await browser.newContext();
|
||||
const page = await ctx.newPage();
|
||||
await page.goto("https://accounts.dmm.com/service/login/password");
|
||||
await page.waitForURL("https://www.dmm.com/", { timeout: 0 });
|
||||
await page.goto("https://www.dmm.com/service/-/exchange");
|
||||
await Promise.race([
|
||||
page.waitForURL("https://www.dmm.com/", { timeout: 0 }),
|
||||
page.waitForURL("https://www.dmm.co.jp/top/", { timeout: 0 }),
|
||||
]);
|
||||
await browser.saveBrowserContext("dmm-books", ctx);
|
||||
},
|
||||
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("dmm-books");
|
||||
|
||||
|
|
@ -194,7 +181,7 @@ export function DmmBooks(browser: Browser) {
|
|||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<string>>> {
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("dmm-books");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
|
|
@ -203,13 +190,23 @@ export function DmmBooks(browser: Browser) {
|
|||
const imageFiles = await page.evaluate(getImageFiles);
|
||||
|
||||
return imageFiles.map((imageFile) => async () => {
|
||||
const dataUrl = await browser.drawImage(page, imageFile);
|
||||
const blob = await browser.drawImage(page, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
return dataUrl;
|
||||
return blob;
|
||||
});
|
||||
},
|
||||
loginEndpoints: [
|
||||
"https://accounts.dmm.com/service/login/password",
|
||||
"https://www.dmm.com/service/-/exchange",
|
||||
],
|
||||
loginSuccessUrl: (url: URL) =>
|
||||
["https://www.dmm.com/", "https://www.dmm.co.jp/top/"].includes(url.href),
|
||||
logoutEndpoints: [
|
||||
"https://accounts.dmm.com/service/logout",
|
||||
"https://accounts.dmm.co.jp/service/logout",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ export function FanzaDoujin(browser: Browser) {
|
|||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
while (pager.page * pager.perPage <= pager.totalCount) {
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
|
|
@ -51,14 +51,6 @@ export function FanzaDoujin(browser: Browser) {
|
|||
}
|
||||
|
||||
return {
|
||||
async login() {
|
||||
const ctx = await browser.newContext();
|
||||
const page = await ctx.newPage();
|
||||
await page.goto("https://accounts.dmm.co.jp/service/login/password");
|
||||
await page.waitForURL("https://www.dmm.co.jp/top/", { timeout: 0 });
|
||||
await browser.saveBrowserContext("fanza-doujin", ctx);
|
||||
},
|
||||
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("fanza-doujin");
|
||||
|
||||
|
|
@ -67,7 +59,7 @@ export function FanzaDoujin(browser: Browser) {
|
|||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<string>>> {
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("fanza-doujin");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
|
|
@ -117,13 +109,16 @@ export function FanzaDoujin(browser: Browser) {
|
|||
}
|
||||
|
||||
return imageFiles.map((imageFile) => async () => {
|
||||
const dataUrl = await browser.drawImage(page, imageFile);
|
||||
const blob = await browser.drawImage(page, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
return dataUrl;
|
||||
return blob;
|
||||
});
|
||||
},
|
||||
loginEndpoints: ["https://accounts.dmm.co.jp/service/login/password"],
|
||||
loginSuccessUrl: (url: URL) => url.href === "https://www.dmm.co.jp/top/",
|
||||
logoutEndpoints: ["https://accounts.dmm.co.jp/service/logout"],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,17 +39,6 @@ async function getImageFiles(): Promise<Array<ImageFile>> {
|
|||
|
||||
export function GooglePlayBooks(browser: Browser) {
|
||||
return {
|
||||
async login() {
|
||||
const ctx = await browser.newContext();
|
||||
const page = await ctx.newPage();
|
||||
await page.goto("https://accounts.google.com");
|
||||
await page.waitForURL(
|
||||
(url) => url.origin === "https://myaccount.google.com",
|
||||
{ timeout: 0 },
|
||||
);
|
||||
await browser.saveBrowserContext("google-play-books", ctx);
|
||||
},
|
||||
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("google-play-books");
|
||||
const page = await ctx.newPage();
|
||||
|
|
@ -76,7 +65,7 @@ export function GooglePlayBooks(browser: Browser) {
|
|||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<string>>> {
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("google-play-books");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
|
|
@ -118,7 +107,7 @@ export function GooglePlayBooks(browser: Browser) {
|
|||
});
|
||||
}
|
||||
|
||||
const fileMap: Map<string, () => Promise<string>> = new Map();
|
||||
const fileMap: Map<string, () => Promise<Blob>> = new Map();
|
||||
|
||||
while (await next()) {
|
||||
const imageFiles = await frame.evaluate(getImageFiles);
|
||||
|
|
@ -126,16 +115,20 @@ export function GooglePlayBooks(browser: Browser) {
|
|||
for (const imageFile of imageFiles) {
|
||||
if (fileMap.has(imageFile.url)) continue;
|
||||
|
||||
const dataUrl = await browser.drawImage(frame, imageFile);
|
||||
const blob = await browser.drawImage(frame, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
fileMap.set(imageFile.url, async () => dataUrl);
|
||||
fileMap.set(imageFile.url, async () => blob);
|
||||
}
|
||||
}
|
||||
|
||||
return [...fileMap.values()];
|
||||
},
|
||||
loginEndpoints: ["https://accounts.google.com"],
|
||||
loginSuccessUrl: (url: URL) =>
|
||||
url.origin === "https://myaccount.google.com",
|
||||
logoutEndpoints: ["https://accounts.google.com/Logout"],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue