nebel
/
gadl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

dlsite-maniax

This commit is contained in:
Nebel 2023-12-03 16:14:38 +09:00
parent 7547a34e03
commit a53de8e5dd
Signed by: nebel
GPG Key ID: 79807D08C6EF6460
11 changed files with 200 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -9,6 +9,7 @@ $ npx https://git.fogtype.com/nebel/gadl/archive/main.tar.gz --help
## Supported Sites
- DLsite 同人
- DMM ブックス (漫画)
- FANZA 同人
- Google Play ブックス (漫画)

35
browser.ts
View File

@ -66,6 +66,11 @@ async function fetchImage(imageFile: ImageFile): Promise<string> {
return dataUrl;
}
async function dataUrlToBlob(dataUrl: string): Promise<Blob> {
const res = await fetch(dataUrl);
return await res.blob();
}
export type Browser = {
loadBrowserContext(platform: TPlatform): Promise<Playwright.BrowserContext>;
saveBrowserContext(platform: TPlatform, ctx: BrowserContext): Promise<void>;
@ -74,7 +79,7 @@ export type Browser = {
drawImage(
pageOrFrame: Playwright.Page | Playwright.Frame,
imageFile: ImageFile,
): Promise<string>;
): Promise<Blob>;
};
export type BrowserContext = Playwright.BrowserContext;
@ -124,22 +129,38 @@ export async function createBrowser({
async drawImage(
pageOrFrame: Playwright.Page | Playwright.Frame,
imageFile: ImageFile,
): Promise<string> {
): Promise<Blob> {
if (Array.isArray(imageFile.blocks) && imageFile.blocks.length > 0) {
return await pageOrFrame.evaluate(drawImage, imageFile);
const dataUrl = await pageOrFrame.evaluate(drawImage, imageFile);
return await dataUrlToBlob(dataUrl);
}
if (imageFile.url.startsWith("blob:")) {
return await pageOrFrame.evaluate(fetchImage, imageFile);
const dataUrl = await pageOrFrame.evaluate(fetchImage, imageFile);
return await dataUrlToBlob(dataUrl);
}
const page = "page" in pageOrFrame ? pageOrFrame.page() : pageOrFrame;
const res = await page.context().request.get(imageFile.url);
const headers = res.headers();
const buffer = await res.body();
const type = res.headers()["content-type"];
const base64 = buffer.toString("base64");
return `data:${type};base64,${base64}`;
let type = headers["content-type"];
if (type === "binary/octet-stream") {
const [, extension] =
/^attachment *; *filename="[^"]+[.]([^.]+)" *(?:$|;)/i.exec(
headers["content-disposition"],
) ?? [];
switch (extension) {
case "zip":
type = "application/zip";
break;
}
}
return new Blob([buffer], { type });
},
};
}

2
main.ts
View File

@ -82,7 +82,7 @@ const options = {
async run() {
const db = await createDatabase(args.values.db!);
const library = createLibrary(db);
const book = await library.get(Number(args.values.view!));
const book = await library.get(args.values.view!);
if (!book) {
process.exit(1);

2
migrations/4_add_dlsite_maniax.sql Normal file
View File

@ -0,0 +1,2 @@
insert into platforms(name) values
('dlsite-maniax');

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@fogtype/gadl",
"version": "1.2.0",
"version": "1.3.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@fogtype/gadl",
"version": "1.2.0",
"version": "1.3.0",
"license": "AGPL-3.0",
"dependencies": {
"fflate": "^0.8.1",

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@fogtype/gadl",
"version": "1.2.0",
"version": "1.3.0",
"license": "AGPL-3.0",
"type": "module",
"bin": "bin/run.js",

44
platform.ts
View File

@ -3,11 +3,13 @@ import path from "node:path";
import type { Book } from "./library";
import type { Browser } from "./browser";
import type { Database } from "./database";
import { DlsiteManiax } from "./platforms/dlsite-maniax";
import { DmmBooks } from "./platforms/dmm-books";
import { FanzaDoujin } from "./platforms/fanza-doujin";
import { GooglePlayBooks } from "./platforms/google-play-books";
const platforms = {
"dlsite-maniax": DlsiteManiax,
"dmm-books": DmmBooks,
"fanza-doujin": FanzaDoujin,
"google-play-books": GooglePlayBooks,
@ -45,7 +47,7 @@ export function createPlatform(opts: {
await fs.mkdir(path.dirname(dir), { recursive: true });
await fs.mkdir(dir);
const files: Array<() => Promise<string>> = await platform.getFiles(book);
const files: Array<() => Promise<Blob>> = await platform.getFiles(book);
const digits = String(files.length).length;
function pad(n: string) {
@ -59,32 +61,48 @@ export function createPlatform(opts: {
"application/vnd.comicbook+zip": "cbz",
};
for (const [n, dataUrl] of Object.entries(files)) {
const [prefix, base64] = (await dataUrl()).split(",", 2);
const [, type, encoding] =
/^data:([^;]*)(;base64)?$/.exec(prefix) ?? [];
const extension = supportedTypes[type];
for (const [n, getBlob] of Object.entries(files)) {
const blob = await getBlob();
const extension = supportedTypes[blob.type];
if (!extension) {
throw new Error(
`It was ${type}. The image must be a file of type: ${[
`It was ${blob.type}. The image must be a file of type: ${[
...Object.keys(supportedTypes),
].join(", ")}.`,
);
}
if (encoding !== ";base64") {
throw new Error("Only base64 is supported.");
}
const buffer = Buffer.from(base64, "base64");
const buffer = Buffer.from(await blob.arrayBuffer());
await fs.writeFile(`${dir}/${pad(n)}.${extension}`, buffer);
}
process.stderr.write(`\n`);
},
async login() {
const ctx = await opts.browser.newContext();
const page = await ctx.newPage();
for (const loginEndpoint of platform.loginEndpoints) {
await page.goto(loginEndpoint);
await page.waitForURL(platform.loginSuccessUrl, { timeout: 0 });
}
await opts.browser.saveBrowserContext(opts.platform, ctx);
},
async logout() {
try {
const ctx = await opts.browser.loadBrowserContext(opts.platform);
const page = await ctx.newPage();
for (const logoutEndpoint of platform.logoutEndpoints) {
await page.goto(logoutEndpoint);
}
} catch (error) {
process.stderr.write(`Warning: ${(error as Error).message}\n`);
}
await opts.db.run(
`update platforms set secrets = 'null' where name = ?`,
opts.platform,

104
platforms/dlsite-maniax.ts Normal file
View File

@ -0,0 +1,104 @@
import type { Book } from "../library";
import type { Browser, BrowserContext, ImageFile } from "../browser";
export function DlsiteManiax(browser: Browser) {
async function* getAllBooks(ctx: BrowserContext): AsyncGenerator<Book> {
const totalCountEndpoint = "https://play.dlsite.com/api/product_count";
const endpoint = "https://play.dlsite.com/api/purchases";
const pager = {
page: 1,
perPage: 50,
totalCount: Infinity,
};
const res = await ctx.request.get(totalCountEndpoint);
const body: {
user: number;
} = await res.json();
pager.totalCount = body.user;
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
const res = await ctx.request.get(`${endpoint}?page=${pager.page}`);
if (!res.ok()) {
throw new Error(`${res.status()} ${res.statusText()}`);
}
const body: {
limit: number;
works: Array<{
workno: number;
name: {
ja_JP: string;
};
maker: {
name: {
ja_JP: string;
};
};
}>;
} = await res.json();
for (const work of Object.values(body.works).flat()) {
yield {
id: NaN,
platform: "dlsite-maniax",
readerUrl: `https://play.dlsite.com/#/work/${work.workno}`,
title: work.name.ja_JP || "",
authors: [work.maker.name.ja_JP || ""],
};
process.stderr.write(".");
}
pager.page += 1;
pager.perPage = body.limit;
}
}
return {
async *pull(): AsyncGenerator<Book> {
const ctx = await browser.loadBrowserContext("dlsite-maniax");
yield* getAllBooks(ctx);
process.stderr.write(`\n`);
},
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
const ctx = await browser.loadBrowserContext("dlsite-maniax");
const page = await ctx.newPage();
await page.goto(book.readerUrl);
const [, workId] =
/^https:[/][/]play[.]dlsite[.]com[/]#[/]work[/]([^/]+)/.exec(
book.readerUrl,
) ?? [];
if (!workId) {
throw new Error(`workId is not included: ${book.readerUrl}`);
}
const url = `https://www.dlsite.com/home/download/=/product_id/${workId}.html`;
const imageFile = { url };
return [
async () => {
const blob = await browser.drawImage(page, imageFile);
process.stderr.write(".");
return blob;
},
];
},
loginEndpoints: ["https://www.dlsite.com/home/login"],
loginSuccessUrl: (url: URL) => url.origin === "https://www.dlsite.com",
logoutEndpoints: ["https://www.dlsite.com/home/logout"],
};
}
DlsiteManiax.siteUrl = (url: URL) =>
url.href.startsWith("https://play.dlsite.com/#/work/");

33
platforms/dmm-books.ts
View File

@ -89,7 +89,7 @@ export function DmmBooks(browser: Browser) {
totalCount: Infinity,
};
while (pager.page * pager.perPage <= pager.totalCount) {
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
if (!res.ok()) {
@ -136,7 +136,7 @@ export function DmmBooks(browser: Browser) {
totalCount: Infinity,
};
while (pager.page * pager.perPage <= pager.totalCount) {
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
if (!res.ok()) {
@ -173,19 +173,6 @@ export function DmmBooks(browser: Browser) {
}
return {
async login() {
const ctx = await browser.newContext();
const page = await ctx.newPage();
await page.goto("https://accounts.dmm.com/service/login/password");
await page.waitForURL("https://www.dmm.com/", { timeout: 0 });
await page.goto("https://www.dmm.com/service/-/exchange");
await Promise.race([
page.waitForURL("https://www.dmm.com/", { timeout: 0 }),
page.waitForURL("https://www.dmm.co.jp/top/", { timeout: 0 }),
]);
await browser.saveBrowserContext("dmm-books", ctx);
},
async *pull(): AsyncGenerator<Book> {
const ctx = await browser.loadBrowserContext("dmm-books");
@ -194,7 +181,7 @@ export function DmmBooks(browser: Browser) {
process.stderr.write(`\n`);
},
async getFiles(book: Book): Promise<Array<() => Promise<string>>> {
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
const ctx = await browser.loadBrowserContext("dmm-books");
const page = await ctx.newPage();
@ -203,13 +190,23 @@ export function DmmBooks(browser: Browser) {
const imageFiles = await page.evaluate(getImageFiles);
return imageFiles.map((imageFile) => async () => {
const dataUrl = await browser.drawImage(page, imageFile);
const blob = await browser.drawImage(page, imageFile);
process.stderr.write(".");
return dataUrl;
return blob;
});
},
loginEndpoints: [
"https://accounts.dmm.com/service/login/password",
"https://www.dmm.com/service/-/exchange",
],
loginSuccessUrl: (url: URL) =>
["https://www.dmm.com/", "https://www.dmm.co.jp/top/"].includes(url.href),
logoutEndpoints: [
"https://accounts.dmm.com/service/logout",
"https://accounts.dmm.co.jp/service/logout",
],
};
}

19
platforms/fanza-doujin.ts
View File

@ -12,7 +12,7 @@ export function FanzaDoujin(browser: Browser) {
totalCount: Infinity,
};
while (pager.page * pager.perPage <= pager.totalCount) {
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
if (!res.ok()) {
@ -51,14 +51,6 @@ export function FanzaDoujin(browser: Browser) {
}
return {
async login() {
const ctx = await browser.newContext();
const page = await ctx.newPage();
await page.goto("https://accounts.dmm.co.jp/service/login/password");
await page.waitForURL("https://www.dmm.co.jp/top/", { timeout: 0 });
await browser.saveBrowserContext("fanza-doujin", ctx);
},
async *pull(): AsyncGenerator<Book> {
const ctx = await browser.loadBrowserContext("fanza-doujin");
@ -67,7 +59,7 @@ export function FanzaDoujin(browser: Browser) {
process.stderr.write(`\n`);
},
async getFiles(book: Book): Promise<Array<() => Promise<string>>> {
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
const ctx = await browser.loadBrowserContext("fanza-doujin");
const page = await ctx.newPage();
@ -117,13 +109,16 @@ export function FanzaDoujin(browser: Browser) {
}
return imageFiles.map((imageFile) => async () => {
const dataUrl = await browser.drawImage(page, imageFile);
const blob = await browser.drawImage(page, imageFile);
process.stderr.write(".");
return dataUrl;
return blob;
});
},
loginEndpoints: ["https://accounts.dmm.co.jp/service/login/password"],
loginSuccessUrl: (url: URL) => url.href === "https://www.dmm.co.jp/top/",
logoutEndpoints: ["https://accounts.dmm.co.jp/service/logout"],
};
}

23
platforms/google-play-books.ts
View File

@ -39,17 +39,6 @@ async function getImageFiles(): Promise<Array<ImageFile>> {
export function GooglePlayBooks(browser: Browser) {
return {
async login() {
const ctx = await browser.newContext();
const page = await ctx.newPage();
await page.goto("https://accounts.google.com");
await page.waitForURL(
(url) => url.origin === "https://myaccount.google.com",
{ timeout: 0 },
);
await browser.saveBrowserContext("google-play-books", ctx);
},
async *pull(): AsyncGenerator<Book> {
const ctx = await browser.loadBrowserContext("google-play-books");
const page = await ctx.newPage();
@ -76,7 +65,7 @@ export function GooglePlayBooks(browser: Browser) {
process.stderr.write(`\n`);
},
async getFiles(book: Book): Promise<Array<() => Promise<string>>> {
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
const ctx = await browser.loadBrowserContext("google-play-books");
const page = await ctx.newPage();
@ -118,7 +107,7 @@ export function GooglePlayBooks(browser: Browser) {
});
}
const fileMap: Map<string, () => Promise<string>> = new Map();
const fileMap: Map<string, () => Promise<Blob>> = new Map();
while (await next()) {
const imageFiles = await frame.evaluate(getImageFiles);
@ -126,16 +115,20 @@ export function GooglePlayBooks(browser: Browser) {
for (const imageFile of imageFiles) {
if (fileMap.has(imageFile.url)) continue;
const dataUrl = await browser.drawImage(frame, imageFile);
const blob = await browser.drawImage(frame, imageFile);
process.stderr.write(".");
fileMap.set(imageFile.url, async () => dataUrl);
fileMap.set(imageFile.url, async () => blob);
}
}
return [...fileMap.values()];
},
loginEndpoints: ["https://accounts.google.com"],
loginSuccessUrl: (url: URL) =>
url.origin === "https://myaccount.google.com",
logoutEndpoints: ["https://accounts.google.com/Logout"],
};
}