cleanup project
This commit is contained in:
parent
445110c552
commit
59c28ca0cd
GPG key ID:
79807D08C6EF6460
14 changed files with 5 additions and 2911 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,3 +1,2 @@
|
|||
node_modules
|
||||
*.db
|
||||
dist
|
||||
|
|
|
|||
12
README.md
12
README.md
|
|
@ -2,16 +2,14 @@
|
|||
|
||||
## Usage
|
||||
|
||||
```
|
||||
$ npx https://git.fogtype.com/nebel/gadl/archive/main.tar.gz --help
|
||||
```
|
||||
_TODO_
|
||||
|
||||
## Supported Sites
|
||||
|
||||
- DLsite 同人
|
||||
- DMM ブックス (漫画)
|
||||
- FANZA 同人
|
||||
- Google Play ブックス (漫画)
|
||||
- [ ] DLsite 同人
|
||||
- [ ] DMM ブックス (漫画)
|
||||
- [ ] FANZA 同人
|
||||
- [ ] Google Play ブックス (漫画)
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
import "tsx";
|
||||
|
||||
await import("../main.ts");
|
||||
166
browser.ts
166
browser.ts
|
|
@ -1,166 +0,0 @@
|
|||
import * as Playwright from "playwright";
|
||||
import { chromium, devices } from "playwright";
|
||||
import type { Database } from "./database";
|
||||
import type { TPlatform } from "./platform";
|
||||
|
||||
export type ImageFile = {
|
||||
url: string;
|
||||
blocks?: Array<Record<string, number>>;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
async function drawImage(imageFile: ImageFile): Promise<string> {
|
||||
const canvas = Object.assign(document.createElement("canvas"), {
|
||||
width: imageFile.width,
|
||||
height: imageFile.height,
|
||||
});
|
||||
|
||||
const image = (await new Promise((resolve) => {
|
||||
Object.assign(new Image(), {
|
||||
crossOrigin: "use-credentials",
|
||||
src: imageFile.url,
|
||||
onload() {
|
||||
resolve(this);
|
||||
},
|
||||
});
|
||||
})) as HTMLImageElement;
|
||||
|
||||
const ctx = canvas.getContext("2d")!;
|
||||
|
||||
for (const q of imageFile.blocks!) {
|
||||
ctx.drawImage(
|
||||
image,
|
||||
q.destX,
|
||||
q.destY,
|
||||
q.width,
|
||||
q.height,
|
||||
q.srcX,
|
||||
q.srcY,
|
||||
q.width,
|
||||
q.height,
|
||||
);
|
||||
}
|
||||
|
||||
const dataUrl = canvas.toDataURL();
|
||||
return dataUrl;
|
||||
}
|
||||
|
||||
async function fetchImage(imageFile: ImageFile): Promise<string> {
|
||||
const res = await fetch(imageFile.url);
|
||||
const blob = await res.blob();
|
||||
const dataUrl: string = await new Promise((resolve, reject) => {
|
||||
const fileReader = Object.assign(new FileReader(), {
|
||||
onload(): void {
|
||||
resolve(this.result);
|
||||
},
|
||||
onerror(e: ErrorEvent): void {
|
||||
const error = new Error(`${e.type}: ${e.message}`);
|
||||
reject(error);
|
||||
},
|
||||
});
|
||||
|
||||
fileReader.readAsDataURL(blob);
|
||||
});
|
||||
|
||||
return dataUrl;
|
||||
}
|
||||
|
||||
async function dataUrlToBlob(dataUrl: string): Promise<Blob> {
|
||||
const res = await fetch(dataUrl);
|
||||
return await res.blob();
|
||||
}
|
||||
|
||||
export type Browser = {
|
||||
loadBrowserContext(platform: TPlatform): Promise<Playwright.BrowserContext>;
|
||||
saveBrowserContext(platform: TPlatform, ctx: BrowserContext): Promise<void>;
|
||||
newContext(): Promise<Playwright.BrowserContext>;
|
||||
close(): Promise<void>;
|
||||
drawImage(
|
||||
pageOrFrame: Playwright.Page | Playwright.Frame,
|
||||
imageFile: ImageFile,
|
||||
): Promise<Blob>;
|
||||
};
|
||||
|
||||
export type BrowserContext = Playwright.BrowserContext;
|
||||
|
||||
export async function createBrowser({
|
||||
db,
|
||||
headless = true,
|
||||
}: {
|
||||
db: Database;
|
||||
headless?: boolean;
|
||||
}): Promise<Browser> {
|
||||
const { userAgent } = devices["Desktop Chrome"];
|
||||
const browser = await chromium.launch({
|
||||
headless,
|
||||
args: ["--disable-blink-features=AutomationControlled"],
|
||||
});
|
||||
|
||||
return {
|
||||
async loadBrowserContext(
|
||||
platform: TPlatform,
|
||||
): Promise<Playwright.BrowserContext> {
|
||||
const { secrets } = await db.get(
|
||||
`select secrets from platforms where name = ?`,
|
||||
platform,
|
||||
);
|
||||
|
||||
const storageState = JSON.parse(secrets) ?? undefined;
|
||||
const ctx = await browser.newContext({ storageState, userAgent });
|
||||
return ctx;
|
||||
},
|
||||
|
||||
async saveBrowserContext(
|
||||
platform: TPlatform,
|
||||
ctx: BrowserContext,
|
||||
): Promise<void> {
|
||||
const secrets = await ctx.storageState();
|
||||
await db.run(
|
||||
`update platforms set secrets = ? where name = ?`,
|
||||
JSON.stringify(secrets),
|
||||
platform,
|
||||
);
|
||||
},
|
||||
|
||||
newContext: () => browser.newContext(),
|
||||
close: () => browser.close(),
|
||||
|
||||
async drawImage(
|
||||
pageOrFrame: Playwright.Page | Playwright.Frame,
|
||||
imageFile: ImageFile,
|
||||
): Promise<Blob> {
|
||||
if (Array.isArray(imageFile.blocks) && imageFile.blocks.length > 0) {
|
||||
const dataUrl = await pageOrFrame.evaluate(drawImage, imageFile);
|
||||
return await dataUrlToBlob(dataUrl);
|
||||
}
|
||||
|
||||
if (imageFile.url.startsWith("blob:")) {
|
||||
const dataUrl = await pageOrFrame.evaluate(fetchImage, imageFile);
|
||||
return await dataUrlToBlob(dataUrl);
|
||||
}
|
||||
|
||||
const page = "page" in pageOrFrame ? pageOrFrame.page() : pageOrFrame;
|
||||
const res = await page.context().request.get(imageFile.url);
|
||||
const headers = res.headers();
|
||||
const buffer = await res.body();
|
||||
|
||||
let type = headers["content-type"];
|
||||
|
||||
if (type === "binary/octet-stream") {
|
||||
const [, extension] =
|
||||
/^attachment *; *filename="[^"]+[.]([^.]+)" *(?:$|;)/i.exec(
|
||||
headers["content-disposition"],
|
||||
) ?? [];
|
||||
|
||||
switch (extension) {
|
||||
case "zip":
|
||||
type = "application/zip";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new Blob([buffer], { type });
|
||||
},
|
||||
};
|
||||
}
|
||||
21
database.ts
21
database.ts
|
|
@ -1,21 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import sqlite3 from "sqlite3";
|
||||
import { Database, open } from "sqlite";
|
||||
|
||||
export { Database };
|
||||
|
||||
export async function createDatabase(file: string): Promise<Database> {
|
||||
await fs.mkdir(path.dirname(file), { recursive: true });
|
||||
|
||||
const db = await open({
|
||||
filename: file,
|
||||
driver: sqlite3.cached.Database,
|
||||
});
|
||||
|
||||
const migrationsPath = new URL("./migrations", import.meta.url).pathname;
|
||||
await fs.chmod(file, 0o600);
|
||||
await db.migrate({ migrationsPath });
|
||||
|
||||
return db;
|
||||
}
|
||||
156
library.ts
156
library.ts
|
|
@ -1,156 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import stream from "node:stream/promises";
|
||||
import { Zip, ZipPassThrough } from "fflate";
|
||||
import { Database } from "./database";
|
||||
import { type TPlatform, site } from "./platform";
|
||||
|
||||
export type Book = {
|
||||
id: number;
|
||||
platform: TPlatform;
|
||||
readerUrl: string;
|
||||
title: string;
|
||||
authors: Array<string>;
|
||||
};
|
||||
|
||||
export function createLibrary(db: Database) {
|
||||
return {
|
||||
async add(readerUrlOrBook: string | Book) {
|
||||
if (typeof readerUrlOrBook === "string") {
|
||||
const platform = site(readerUrlOrBook);
|
||||
|
||||
await db.run(
|
||||
`\
|
||||
insert into books(
|
||||
platform_id,
|
||||
reader_url)
|
||||
values((select id from platforms where name = ?), ?)
|
||||
on conflict(reader_url)
|
||||
do nothing
|
||||
`,
|
||||
platform,
|
||||
readerUrlOrBook,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await db.run(
|
||||
`\
|
||||
insert into books(
|
||||
platform_id,
|
||||
reader_url,
|
||||
title,
|
||||
authors)
|
||||
values((select id from platforms where name = ?), ?, ?, ?)
|
||||
on conflict(reader_url)
|
||||
do update set title = excluded.title, authors = excluded.authors
|
||||
`,
|
||||
readerUrlOrBook.platform,
|
||||
readerUrlOrBook.readerUrl,
|
||||
readerUrlOrBook.title,
|
||||
JSON.stringify(readerUrlOrBook.authors),
|
||||
);
|
||||
},
|
||||
async delete(id: number) {
|
||||
await db.run(`delete from books where id = ?`, id);
|
||||
},
|
||||
async get(readerUrlOrBookId: string | number): Promise<Book | undefined> {
|
||||
const row = await db.get(
|
||||
`select books.id, platforms.name as platform, books.reader_url as readerUrl, books.title, books.authors from books left join platforms on books.platform_id = platforms.id where books.reader_url = ? or books.id = ?`,
|
||||
readerUrlOrBookId,
|
||||
Number(readerUrlOrBookId),
|
||||
);
|
||||
|
||||
const book: Book | undefined = row && {
|
||||
...row,
|
||||
authors: JSON.parse(row.authors),
|
||||
};
|
||||
|
||||
return book;
|
||||
},
|
||||
async getBooks(): Promise<Array<Book>> {
|
||||
const rows = await db.all(
|
||||
`select books.id, platforms.name as platform, books.reader_url as readerUrl, books.title, books.authors from books left join platforms on books.platform_id = platforms.id`,
|
||||
);
|
||||
|
||||
const books: Array<Book> = rows.map((row) => ({
|
||||
...row,
|
||||
authors: JSON.parse(row.authors),
|
||||
}));
|
||||
|
||||
return books;
|
||||
},
|
||||
async archive(
|
||||
path: string,
|
||||
book: Book,
|
||||
opts: {
|
||||
outDir: string;
|
||||
outAuthorsLimit: number;
|
||||
},
|
||||
) {
|
||||
const bookDir = await fs.stat(path);
|
||||
|
||||
if (!bookDir.isDirectory()) {
|
||||
throw new Error(`Not found: ${path}`);
|
||||
}
|
||||
|
||||
if (!book.title) {
|
||||
book.title = String(book.id);
|
||||
}
|
||||
|
||||
if (book.authors.length > 0) {
|
||||
book.title = `「${book.title}」`;
|
||||
}
|
||||
|
||||
const title = `${book.authors.slice(0, opts.outAuthorsLimit).join("、")}${
|
||||
book.title
|
||||
}`.replace(/[/]/g, "%2F");
|
||||
|
||||
const files = await fs.readdir(path);
|
||||
|
||||
if (files.every((f) => f.match(/[.](zip|cbz)$/))) {
|
||||
const digits = String(files.length).length;
|
||||
|
||||
function pad(n: string) {
|
||||
return n.padStart(digits, "0");
|
||||
}
|
||||
|
||||
for (const [n, f] of Object.entries(files)) {
|
||||
await fs.rename(
|
||||
`${path}/${f}`,
|
||||
`${opts.outDir}/${title}${
|
||||
files.length > 1 ? ` - ${pad(n)}` : ""
|
||||
}.${f.split(".").at(-1)}`,
|
||||
);
|
||||
}
|
||||
await fs.rmdir(path);
|
||||
return;
|
||||
}
|
||||
|
||||
const out = createWriteStream(`${opts.outDir}/${title}.cbz`);
|
||||
|
||||
const zip = new Zip(function cb(err, data, final) {
|
||||
if (err) {
|
||||
out.destroy(err);
|
||||
return;
|
||||
}
|
||||
|
||||
out[final ? "end" : "write"](data);
|
||||
});
|
||||
|
||||
for (const file of files) {
|
||||
const data = new ZipPassThrough(file);
|
||||
zip.add(data);
|
||||
|
||||
const buffer = await fs.readFile(`${path}/${file}`);
|
||||
data.push(buffer, true);
|
||||
}
|
||||
|
||||
zip.end();
|
||||
|
||||
await stream.finished(out);
|
||||
await fs.rm(path, { recursive: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
246
main.ts
246
main.ts
|
|
@ -1,246 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import util from "node:util";
|
||||
import { createBrowser } from "./browser";
|
||||
import { createDatabase } from "./database";
|
||||
import { type Book, createLibrary } from "./library";
|
||||
import { type TPlatform, createPlatform, platforms } from "./platform";
|
||||
import * as pkg from "./package.json";
|
||||
|
||||
const options = {
|
||||
db: {
|
||||
type: "string",
|
||||
default: "gadl.db",
|
||||
toString() {
|
||||
return `<database_file_path> (default: ${this.default})`;
|
||||
},
|
||||
},
|
||||
"out-dir": {
|
||||
type: "string",
|
||||
default: "dist",
|
||||
toString() {
|
||||
return `<output_directory_path> (default: ${this.default})`;
|
||||
},
|
||||
},
|
||||
"out-authors-limit": {
|
||||
type: "string",
|
||||
default: "3",
|
||||
toString() {
|
||||
return `<output_authors_limit> (default: ${this.default})`;
|
||||
},
|
||||
},
|
||||
login: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return [...Object.keys(platforms)].join("|");
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const browser = await createBrowser({ db, headless: false });
|
||||
const platform = createPlatform({
|
||||
platform: args.values.login as TPlatform,
|
||||
db,
|
||||
browser,
|
||||
});
|
||||
await platform.login();
|
||||
await browser.close();
|
||||
},
|
||||
},
|
||||
logout: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return [...Object.keys(platforms)].join("|");
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const browser = await createBrowser({ db });
|
||||
const platform = createPlatform({
|
||||
platform: args.values.logout as TPlatform,
|
||||
db,
|
||||
browser,
|
||||
});
|
||||
await platform.logout();
|
||||
await browser.close();
|
||||
},
|
||||
},
|
||||
add: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return `<reader_url_or_id>`;
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
await library.add(args.values.add!);
|
||||
},
|
||||
},
|
||||
delete: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return `<id>`;
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
await library.delete(Number(args.values.delete));
|
||||
},
|
||||
},
|
||||
list: {
|
||||
type: "boolean",
|
||||
short: "l",
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
const books = await library.getBooks();
|
||||
|
||||
console.dir(books, {
|
||||
depth: null,
|
||||
maxArrayLength: null,
|
||||
maxStringLength: null,
|
||||
});
|
||||
},
|
||||
},
|
||||
view: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return `<reader_url_or_id>`;
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
const book = await library.get(args.values.view!);
|
||||
|
||||
if (!book) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.dir(book, {
|
||||
depth: null,
|
||||
maxArrayLength: null,
|
||||
maxStringLength: null,
|
||||
});
|
||||
},
|
||||
},
|
||||
pull: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return [...Object.keys(platforms)].join("|");
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
const browser = await createBrowser({ db });
|
||||
const platform = createPlatform({
|
||||
platform: args.values.pull as TPlatform,
|
||||
db,
|
||||
browser,
|
||||
});
|
||||
|
||||
for await (const book of platform.pull()) {
|
||||
await library.add(book);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
},
|
||||
},
|
||||
download: {
|
||||
type: "string",
|
||||
toString() {
|
||||
return `all|<reader_url_or_id>`;
|
||||
},
|
||||
async run() {
|
||||
const db = await createDatabase(args.values.db!);
|
||||
const library = createLibrary(db);
|
||||
const books: Array<Book> = [];
|
||||
|
||||
if (args.values.download === "all") {
|
||||
books.push(...(await library.getBooks()));
|
||||
} else {
|
||||
if (URL.canParse(args.values.download!)) {
|
||||
await library.add(args.values.download!);
|
||||
}
|
||||
|
||||
const book = await library.get(args.values.download!);
|
||||
|
||||
if (!book) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
books.push(book);
|
||||
}
|
||||
|
||||
for (const book of books) {
|
||||
const browser = await createBrowser({ db });
|
||||
const platform = createPlatform({
|
||||
platform: book.platform,
|
||||
db,
|
||||
browser,
|
||||
});
|
||||
const dir = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), `gadl-${book.id}-`),
|
||||
);
|
||||
await platform.download(dir, book);
|
||||
await library.archive(dir, book, {
|
||||
outDir: args.values["out-dir"]!,
|
||||
outAuthorsLimit: Number(args.values["out-authors-limit"]!),
|
||||
});
|
||||
await browser.close();
|
||||
}
|
||||
},
|
||||
},
|
||||
json: {
|
||||
type: "boolean",
|
||||
},
|
||||
version: {
|
||||
type: "boolean",
|
||||
short: "v",
|
||||
run() {
|
||||
console.log(pkg.version);
|
||||
},
|
||||
},
|
||||
help: {
|
||||
type: "boolean",
|
||||
short: "h",
|
||||
run() {
|
||||
console.log(
|
||||
[
|
||||
"Usage: gadl [options...]",
|
||||
` $ npx playwright@${pkg.dependencies.playwright} install --with-deps chromium`,
|
||||
` $ gadl --login=<platform_type>`,
|
||||
` $ gadl --download=<reader_url>`,
|
||||
"",
|
||||
"Available options:",
|
||||
...Object.entries(options).map((option) =>
|
||||
[
|
||||
` --${option[0]}`,
|
||||
"short" in option[1] && ` -${option[1].short}`,
|
||||
option[1].type === "string" && `=${option[1]}`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(""),
|
||||
),
|
||||
].join("\n"),
|
||||
);
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const args = util.parseArgs({ options });
|
||||
|
||||
if (args.values.json) {
|
||||
console.dir = function dir(arrayOrObject) {
|
||||
for (const obj of [arrayOrObject].flat()) {
|
||||
console.log(JSON.stringify(obj));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
for (const option of Object.keys(options)) {
|
||||
if (args.values[option] && typeof options[option].run === "function") {
|
||||
await options[option].run();
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
|
||||
options.help.run();
|
||||
1600
package-lock.json
generated
1600
package-lock.json
generated
File diff suppressed because it is too large
Load diff
17
package.json
17
package.json
|
|
@ -1,17 +0,0 @@
|
|||
{
|
||||
"name": "@fogtype/gadl",
|
||||
"version": "1.4.0",
|
||||
"license": "AGPL-3.0",
|
||||
"type": "module",
|
||||
"bin": "bin/run.js",
|
||||
"dependencies": {
|
||||
"fflate": "^0.8.1",
|
||||
"playwright": "1.40.1",
|
||||
"sqlite": "^5.1.1",
|
||||
"sqlite3": "^5.1.6",
|
||||
"tsx": "^4.6.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.10.2"
|
||||
}
|
||||
}
|
||||
111
platform.ts
111
platform.ts
|
|
@ -1,111 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { Book } from "./library";
|
||||
import type { Browser } from "./browser";
|
||||
import type { Database } from "./database";
|
||||
import { DlsiteManiax } from "./platforms/dlsite-maniax";
|
||||
import { DmmBooks } from "./platforms/dmm-books";
|
||||
import { FanzaDoujin } from "./platforms/fanza-doujin";
|
||||
import { GooglePlayBooks } from "./platforms/google-play-books";
|
||||
|
||||
export const platforms = {
|
||||
"dlsite-maniax": DlsiteManiax,
|
||||
"dmm-books": DmmBooks,
|
||||
"fanza-doujin": FanzaDoujin,
|
||||
"google-play-books": GooglePlayBooks,
|
||||
};
|
||||
|
||||
export type TPlatform = keyof typeof platforms;
|
||||
|
||||
export function site(url: string): TPlatform {
|
||||
for (const [platform, { siteUrl }] of Object.entries(platforms)) {
|
||||
if (siteUrl(new URL(url))) return platform as TPlatform;
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported URL: ${url}`);
|
||||
}
|
||||
|
||||
export function createPlatform(opts: {
|
||||
platform: TPlatform;
|
||||
db: Database;
|
||||
browser: Browser;
|
||||
}) {
|
||||
if (!(opts.platform in platforms)) {
|
||||
throw new Error(
|
||||
`The value must be a platform type: ${[...Object.keys(platforms)].join(
|
||||
", ",
|
||||
)}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const platform = platforms[opts.platform](opts.browser);
|
||||
|
||||
return {
|
||||
...platform,
|
||||
|
||||
async download(dir: string, book: Book): Promise<void> {
|
||||
await fs.mkdir(dir);
|
||||
|
||||
const files: Array<() => Promise<Blob>> = await platform.getFiles(book);
|
||||
const digits = String(files.length).length;
|
||||
|
||||
function pad(n: string) {
|
||||
return n.padStart(digits, "0");
|
||||
}
|
||||
|
||||
const supportedTypes = {
|
||||
"image/png": "png",
|
||||
"image/jpeg": "jpg",
|
||||
"application/zip": "zip",
|
||||
"application/vnd.comicbook+zip": "cbz",
|
||||
};
|
||||
|
||||
for (const [n, getBlob] of Object.entries(files)) {
|
||||
const blob = await getBlob();
|
||||
const extension = supportedTypes[blob.type];
|
||||
if (!extension) {
|
||||
throw new Error(
|
||||
`It was ${blob.type}. The image must be a file of type: ${[
|
||||
...Object.keys(supportedTypes),
|
||||
].join(", ")}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await blob.arrayBuffer());
|
||||
await fs.writeFile(`${dir}/${pad(n)}.${extension}`, buffer);
|
||||
}
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async login() {
|
||||
const ctx = await opts.browser.newContext();
|
||||
const page = await ctx.newPage();
|
||||
|
||||
for (const loginEndpoint of platform.loginEndpoints) {
|
||||
await page.goto(loginEndpoint);
|
||||
await page.waitForURL(platform.loginSuccessUrl, { timeout: 0 });
|
||||
}
|
||||
|
||||
await opts.browser.saveBrowserContext(opts.platform, ctx);
|
||||
},
|
||||
|
||||
async logout() {
|
||||
try {
|
||||
const ctx = await opts.browser.loadBrowserContext(opts.platform);
|
||||
const page = await ctx.newPage();
|
||||
|
||||
for (const logoutEndpoint of platform.logoutEndpoints) {
|
||||
await page.goto(logoutEndpoint);
|
||||
}
|
||||
} catch (error) {
|
||||
process.stderr.write(`Warning: ${(error as Error).message}\n`);
|
||||
}
|
||||
|
||||
await opts.db.run(
|
||||
`update platforms set secrets = 'null' where name = ?`,
|
||||
opts.platform,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
import type { Book } from "../library";
|
||||
import type { Browser, BrowserContext, ImageFile } from "../browser";
|
||||
|
||||
export function DlsiteManiax(browser: Browser) {
|
||||
async function* getAllBooks(ctx: BrowserContext): AsyncGenerator<Book> {
|
||||
const totalCountEndpoint = "https://play.dlsite.com/api/product_count";
|
||||
const endpoint = "https://play.dlsite.com/api/purchases";
|
||||
const pager = {
|
||||
page: 1,
|
||||
perPage: 50,
|
||||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
const res = await ctx.request.get(totalCountEndpoint);
|
||||
const body: {
|
||||
user: number;
|
||||
} = await res.json();
|
||||
|
||||
pager.totalCount = body.user;
|
||||
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}?page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const body: {
|
||||
limit: number;
|
||||
works: Array<{
|
||||
workno: number;
|
||||
name: {
|
||||
ja_JP: string;
|
||||
};
|
||||
maker: {
|
||||
name: {
|
||||
ja_JP: string;
|
||||
};
|
||||
};
|
||||
}>;
|
||||
} = await res.json();
|
||||
|
||||
for (const work of Object.values(body.works).flat()) {
|
||||
yield {
|
||||
id: NaN,
|
||||
platform: "dlsite-maniax",
|
||||
readerUrl: `https://play.dlsite.com/#/work/${work.workno}`,
|
||||
title: work.name.ja_JP || "",
|
||||
authors: [work.maker.name.ja_JP || ""],
|
||||
};
|
||||
|
||||
process.stderr.write(".");
|
||||
}
|
||||
|
||||
pager.page += 1;
|
||||
pager.perPage = body.limit;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("dlsite-maniax");
|
||||
|
||||
yield* getAllBooks(ctx);
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("dlsite-maniax");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
await page.goto(book.readerUrl);
|
||||
|
||||
const [, workId] =
|
||||
/^https:[/][/]play[.]dlsite[.]com[/]#[/]work[/]([^/]+)/.exec(
|
||||
book.readerUrl,
|
||||
) ?? [];
|
||||
|
||||
if (!workId) {
|
||||
throw new Error(`workId is not included: ${book.readerUrl}`);
|
||||
}
|
||||
|
||||
const url = `https://www.dlsite.com/home/download/=/product_id/${workId}.html`;
|
||||
const imageFile = { url };
|
||||
|
||||
return [
|
||||
async () => {
|
||||
const blob = await browser.drawImage(page, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
return blob;
|
||||
},
|
||||
];
|
||||
},
|
||||
loginEndpoints: ["https://www.dlsite.com/home/login"],
|
||||
loginSuccessUrl: (url: URL) => url.origin === "https://www.dlsite.com",
|
||||
logoutEndpoints: ["https://www.dlsite.com/home/logout"],
|
||||
};
|
||||
}
|
||||
|
||||
DlsiteManiax.siteUrl = (url: URL) =>
|
||||
url.href.startsWith("https://play.dlsite.com/#/work/");
|
||||
|
|
@ -1,214 +0,0 @@
|
|||
import type { Book } from "../library";
|
||||
import type { Browser, BrowserContext, ImageFile } from "../browser";
|
||||
|
||||
var NFBR: any;
|
||||
|
||||
export async function getImageFiles(): Promise<Array<ImageFile>> {
|
||||
const params = new URLSearchParams(location.search);
|
||||
const model = new NFBR.a6G.Model({
|
||||
settings: new NFBR.Settings("NFBR.SettingData"),
|
||||
viewerFontSize: NFBR.a0X.a3K,
|
||||
viewerFontFace: NFBR.a0X.a3k,
|
||||
viewerSpreadDouble: true,
|
||||
viewerSpread: {},
|
||||
});
|
||||
const a6l = new NFBR.a6G.a6L(model);
|
||||
const a2f = new NFBR.a2F();
|
||||
const a5w = await a2f.a5W({
|
||||
contentId: params.get(NFBR.a5q.Key.CONTENT_ID),
|
||||
a6m: params.get(NFBR.a5q.Key.a6M),
|
||||
preview:
|
||||
params.get(NFBR.a5q.Key.LOOK_INSIDE) !== NFBR.a5q.LookInsideType.DISABLED,
|
||||
previewType:
|
||||
params.get(NFBR.a5q.Key.LOOK_INSIDE) ?? NFBR.a5q.LookInsideType.DISABLED,
|
||||
contentType: a6l.getContentType(),
|
||||
title: true,
|
||||
});
|
||||
const content = new NFBR.a6i.Content(a5w.url);
|
||||
const a5n = new NFBR.a5n();
|
||||
await a5n.a5s(content, "configuration", a6l);
|
||||
|
||||
const imageFiles: Array<ImageFile> = [];
|
||||
|
||||
for (const index of Object.keys(content.files)) {
|
||||
const file = content.files[index];
|
||||
const conf = content.configuration.contents[index];
|
||||
const {
|
||||
No,
|
||||
DummyWidth,
|
||||
DummyHeight,
|
||||
Size: { Width, Height },
|
||||
} = file.FileLinkInfo.PageLinkInfoList[0].Page;
|
||||
|
||||
const page = new NFBR.a6i.Page(
|
||||
`${conf.file}/${No}.jpeg`,
|
||||
index,
|
||||
`${conf["original-file-path"]}#-acs-position-${file.PageToBookmark[0][0]}-${file.PageToBookmark[0][1]}`,
|
||||
);
|
||||
|
||||
const w = [...`${conf.file}/${No}`]
|
||||
.map((c) => c.charCodeAt(0))
|
||||
.reduce((a, cc) => a + cc, 0);
|
||||
|
||||
const pattern = (w % NFBR.a0X.a3h) + 1;
|
||||
const blocks = NFBR.a3E.a3f(
|
||||
Width + DummyWidth,
|
||||
Height + DummyHeight,
|
||||
NFBR.a0X.a3g,
|
||||
NFBR.a0X.a3G,
|
||||
pattern,
|
||||
);
|
||||
|
||||
const url = `${a5w.url}${page.url}`;
|
||||
|
||||
imageFiles.push({
|
||||
url,
|
||||
blocks,
|
||||
width: Width,
|
||||
height: Height,
|
||||
});
|
||||
}
|
||||
|
||||
return imageFiles;
|
||||
}
|
||||
|
||||
export function DmmBooks(browser: Browser) {
|
||||
async function* getSeriesBooks(
|
||||
ctx: BrowserContext,
|
||||
series: {
|
||||
seriesId: string;
|
||||
shopName: string;
|
||||
title: string;
|
||||
authors: Array<string>;
|
||||
},
|
||||
): AsyncGenerator<Book> {
|
||||
const endpoint = `https://book.dmm.com/ajax/bff/contents/?shop_name=${series.shopName}&series_id=${series.seriesId}`;
|
||||
const pager = {
|
||||
page: 1,
|
||||
perPage: 0,
|
||||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const body: {
|
||||
volume_books: Array<{
|
||||
title: string;
|
||||
purchased?: {
|
||||
streaming_url: string;
|
||||
};
|
||||
}>;
|
||||
pager: {
|
||||
page: number;
|
||||
per_page: number;
|
||||
total_count: number;
|
||||
};
|
||||
} = await res.json();
|
||||
|
||||
for (const book of body.volume_books.filter((b) => b.purchased)) {
|
||||
yield {
|
||||
id: NaN,
|
||||
platform: "dmm-books",
|
||||
readerUrl: book.purchased?.streaming_url!,
|
||||
title: book.title || series.title || "",
|
||||
authors: series.authors,
|
||||
};
|
||||
|
||||
process.stderr.write(".");
|
||||
}
|
||||
|
||||
pager.page += 1;
|
||||
pager.perPage = body.pager.per_page;
|
||||
pager.totalCount = body.pager.total_count;
|
||||
}
|
||||
}
|
||||
|
||||
async function* getAllBooks(ctx: BrowserContext): AsyncGenerator<Book> {
|
||||
const endpoint = "https://book.dmm.com/ajax/bff/library/?shop_name=all";
|
||||
const pager = {
|
||||
page: 1,
|
||||
perPage: 0,
|
||||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const body: {
|
||||
series_books: Array<{
|
||||
shop_name: string;
|
||||
series_id: string;
|
||||
title: string;
|
||||
author: Array<string>;
|
||||
}>;
|
||||
pager: {
|
||||
page: number;
|
||||
per_page: number;
|
||||
total_count: number;
|
||||
};
|
||||
} = await res.json();
|
||||
|
||||
for (const series of body.series_books) {
|
||||
yield* getSeriesBooks(ctx, {
|
||||
seriesId: series.series_id,
|
||||
shopName: series.shop_name,
|
||||
title: series.title,
|
||||
authors: series.author,
|
||||
});
|
||||
}
|
||||
|
||||
pager.page += 1;
|
||||
pager.perPage = body.pager.per_page;
|
||||
pager.totalCount = body.pager.total_count;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("dmm-books");
|
||||
|
||||
yield* getAllBooks(ctx);
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("dmm-books");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
await page.goto(book.readerUrl);
|
||||
|
||||
const imageFiles = await page.evaluate(getImageFiles);
|
||||
|
||||
return imageFiles.map((imageFile) => async () => {
|
||||
const blob = await browser.drawImage(page, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
return blob;
|
||||
});
|
||||
},
|
||||
loginEndpoints: [
|
||||
"https://accounts.dmm.com/service/login/password",
|
||||
"https://www.dmm.com/service/-/exchange",
|
||||
],
|
||||
loginSuccessUrl: (url: URL) =>
|
||||
["https://www.dmm.com/", "https://www.dmm.co.jp/top/"].includes(url.href),
|
||||
logoutEndpoints: [
|
||||
"https://accounts.dmm.com/service/logout",
|
||||
"https://accounts.dmm.co.jp/service/logout",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
DmmBooks.siteUrl = (url: URL) =>
|
||||
["https://book.dmm.com", "https://book.dmm.co.jp"].includes(url.origin);
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
import type { Book } from "../library";
|
||||
import type { Browser, BrowserContext, ImageFile } from "../browser";
|
||||
import { getImageFiles } from "./dmm-books";
|
||||
|
||||
export function FanzaDoujin(browser: Browser) {
|
||||
async function* getAllBooks(ctx: BrowserContext): AsyncGenerator<Book> {
|
||||
const endpoint =
|
||||
"https://www.dmm.co.jp/dc/doujin/api/mylibraries/?limit=20";
|
||||
const pager = {
|
||||
page: 1,
|
||||
perPage: 20,
|
||||
totalCount: Infinity,
|
||||
};
|
||||
|
||||
while ((pager.page - 1) * pager.perPage <= pager.totalCount) {
|
||||
const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const body: {
|
||||
data: {
|
||||
items: Record<
|
||||
string,
|
||||
Array<{
|
||||
productId: string;
|
||||
title: string;
|
||||
makerName: string;
|
||||
}>
|
||||
>;
|
||||
total: number;
|
||||
};
|
||||
} = await res.json();
|
||||
|
||||
for (const item of Object.values(body.data.items).flat()) {
|
||||
yield {
|
||||
id: NaN,
|
||||
platform: "fanza-doujin",
|
||||
readerUrl: `https://www.dmm.co.jp/dc/-/mylibrary/detail/=/product_id=${item.productId}/`,
|
||||
title: item.title || "",
|
||||
authors: [item.makerName],
|
||||
};
|
||||
|
||||
process.stderr.write(".");
|
||||
}
|
||||
|
||||
pager.page += 1;
|
||||
pager.totalCount = body.data.total;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("fanza-doujin");
|
||||
|
||||
yield* getAllBooks(ctx);
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("fanza-doujin");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
await page.goto(book.readerUrl);
|
||||
|
||||
const [, productId] = /product_id=([^/]*)/.exec(book.readerUrl) ?? [];
|
||||
|
||||
if (!productId) {
|
||||
throw new Error(`product_id is not included: ${book.readerUrl}`);
|
||||
}
|
||||
|
||||
const res = await ctx.request.get(
|
||||
`https://www.dmm.co.jp/dc/doujin/api/mylibraries/details/${productId}/`,
|
||||
);
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const body: {
|
||||
data: {
|
||||
drm: {
|
||||
dmmBooks: boolean;
|
||||
softDenchi: boolean;
|
||||
};
|
||||
downloadLinks: Record<number, string>;
|
||||
};
|
||||
} = await res.json();
|
||||
|
||||
const imageFiles: Array<ImageFile> = [];
|
||||
|
||||
if (body.data.drm.dmmBooks) {
|
||||
await page.waitForSelector(`li[class^="fileTreeItem"]`);
|
||||
await page.click(`li[class^="fileTreeItem"]>a`);
|
||||
await page.waitForURL((url) =>
|
||||
url.href.startsWith(
|
||||
"https://www.dmm.co.jp/dc/-/viewer/=/product_id=",
|
||||
),
|
||||
);
|
||||
|
||||
imageFiles.push(...(await page.evaluate(getImageFiles)));
|
||||
} else {
|
||||
for (const link of Object.values(body.data.downloadLinks)) {
|
||||
const url = new URL(link, "https://www.dmm.co.jp/").href;
|
||||
imageFiles.push({ url });
|
||||
}
|
||||
}
|
||||
|
||||
return imageFiles.map((imageFile) => async () => {
|
||||
const blob = await browser.drawImage(page, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
return blob;
|
||||
});
|
||||
},
|
||||
loginEndpoints: ["https://accounts.dmm.co.jp/service/login/password"],
|
||||
loginSuccessUrl: (url: URL) => url.href === "https://www.dmm.co.jp/top/",
|
||||
logoutEndpoints: ["https://accounts.dmm.co.jp/service/logout"],
|
||||
};
|
||||
}
|
||||
|
||||
FanzaDoujin.siteUrl = (url: URL) =>
|
||||
url.href.startsWith(
|
||||
"https://www.dmm.co.jp/dc/-/mylibrary/detail/=/product_id=",
|
||||
);
|
||||
|
|
@ -1,136 +0,0 @@
|
|||
import type { Book } from "../library";
|
||||
import type { Browser, ImageFile } from "../browser";
|
||||
|
||||
async function getImageFiles(): Promise<Array<ImageFile>> {
|
||||
const pages: NodeListOf<HTMLElement> = await new Promise(async function (
|
||||
resolve,
|
||||
reject,
|
||||
) {
|
||||
const timeout = setTimeout(() => {
|
||||
reject(new Error("Page loading timeout."));
|
||||
}, 60_000);
|
||||
|
||||
let pages: NodeListOf<HTMLElement>;
|
||||
|
||||
while (true) {
|
||||
pages = document.querySelectorAll("reader-page");
|
||||
|
||||
const loaded =
|
||||
pages.length > 0 &&
|
||||
[...pages].every((page) => page.classList.contains("-gb-loaded"));
|
||||
|
||||
if (loaded) {
|
||||
break;
|
||||
} else {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
|
||||
resolve(pages);
|
||||
clearTimeout(timeout);
|
||||
});
|
||||
|
||||
const images: Array<SVGImageElement> = [...pages].map(
|
||||
(el) => el.querySelector("svg image")!,
|
||||
);
|
||||
|
||||
return [...images].map((image) => ({ url: image.href.baseVal }));
|
||||
}
|
||||
|
||||
export function GooglePlayBooks(browser: Browser) {
|
||||
return {
|
||||
async *pull(): AsyncGenerator<Book> {
|
||||
const ctx = await browser.loadBrowserContext("google-play-books");
|
||||
const page = await ctx.newPage();
|
||||
await page.goto(
|
||||
"https://play.google.com/books?type=comics&source=purchases",
|
||||
);
|
||||
await page.waitForSelector("gpb-library-card");
|
||||
|
||||
for (const metadata of await page.$$("gpb-library-card .metadata")) {
|
||||
const readerUrl = await metadata.$eval("a", (a) => a.href);
|
||||
const [title, author] = (await metadata.innerText()).split("\n");
|
||||
|
||||
yield {
|
||||
id: NaN,
|
||||
platform: "google-play-books",
|
||||
readerUrl,
|
||||
title,
|
||||
authors: [author],
|
||||
};
|
||||
|
||||
process.stderr.write(".");
|
||||
}
|
||||
|
||||
process.stderr.write(`\n`);
|
||||
},
|
||||
|
||||
async getFiles(book: Book): Promise<Array<() => Promise<Blob>>> {
|
||||
const ctx = await browser.loadBrowserContext("google-play-books");
|
||||
const page = await ctx.newPage();
|
||||
|
||||
await page.goto(book.readerUrl);
|
||||
await page.waitForSelector(".display");
|
||||
|
||||
const frame = page.frames().at(-1);
|
||||
|
||||
if (!frame) {
|
||||
throw new Error("Frame not found.");
|
||||
}
|
||||
|
||||
await frame.evaluate(function scrollToTop() {
|
||||
const viewport = document.querySelector("cdk-virtual-scroll-viewport");
|
||||
viewport?.scroll({ top: 0 });
|
||||
});
|
||||
|
||||
async function next(): Promise<boolean> {
|
||||
return await frame!.evaluate(function scroll() {
|
||||
const viewport = document.querySelector(
|
||||
"cdk-virtual-scroll-viewport",
|
||||
);
|
||||
|
||||
if (!viewport) throw new Error("Viewport not found.");
|
||||
|
||||
const hasNext =
|
||||
1 <=
|
||||
Math.abs(
|
||||
viewport.scrollHeight -
|
||||
viewport.clientHeight -
|
||||
viewport.scrollTop,
|
||||
);
|
||||
|
||||
if (hasNext) {
|
||||
viewport.scrollBy({ top: viewport.clientHeight });
|
||||
}
|
||||
|
||||
return hasNext;
|
||||
});
|
||||
}
|
||||
|
||||
const fileMap: Map<string, () => Promise<Blob>> = new Map();
|
||||
|
||||
while (await next()) {
|
||||
const imageFiles = await frame.evaluate(getImageFiles);
|
||||
|
||||
for (const imageFile of imageFiles) {
|
||||
if (fileMap.has(imageFile.url)) continue;
|
||||
|
||||
const blob = await browser.drawImage(frame, imageFile);
|
||||
|
||||
process.stderr.write(".");
|
||||
|
||||
fileMap.set(imageFile.url, async () => blob);
|
||||
}
|
||||
}
|
||||
|
||||
return [...fileMap.values()];
|
||||
},
|
||||
loginEndpoints: ["https://accounts.google.com"],
|
||||
loginSuccessUrl: (url: URL) =>
|
||||
url.origin === "https://myaccount.google.com",
|
||||
logoutEndpoints: ["https://accounts.google.com/Logout"],
|
||||
};
|
||||
}
|
||||
|
||||
GooglePlayBooks.siteUrl = (url: URL) =>
|
||||
url.origin === "https://play.google.com";
|
||||
Loading…
Add table
Reference in a new issue