pull

library.ts

@@ -8,35 +8,68 @@ export type Book = {
   id: number;
   platform: "dmm-books" | "google-play-books";
   readerUrl: string;
+  title: string;
+  authors: Array<string>;
 };
 
 export function createLibrary(db: Database) {
   return {
-    async add(readerUrl: string) {
+    async add(readerUrlOrBook: string | Book) {
       const platform = "dmm-books";
 
+      if (typeof readerUrlOrBook === "string") {
         await db.run(
-        `insert into books(reader_url, platform_id) values(?, (select id from platforms where name = ?))`,
+          `insert into books(platform_id, reader_url) values((select id from platforms where name = ?), ?)`,
-        readerUrl,
           platform,
+          readerUrlOrBook,
+        );
+
+        return;
+      }
+
+      await db.run(
+        `\
+insert into books(
+  platform_id,
+  reader_url,
+  title,
+  authors)
+values((select id from platforms where name = ?), ?, ?, ?)
+on conflict(reader_url)
+  do update set title = excluded.title, authors = excluded.authors
+`,
+        platform,
+        readerUrlOrBook.readerUrl,
+        readerUrlOrBook.title,
+        JSON.stringify(readerUrlOrBook.authors),
       );
     },
     async delete(id: number) {
       await db.run(`delete from books where id = ?`, id);
     },
     async get(id: number): Promise<Book | undefined> {
-      const book: Book | undefined = await db.get(
+      const row = await db.get(
-        `select books.id, platforms.name as platform, books.reader_url as readerUrl from books left join platforms on books.platform_id = platforms.id where books.id = ?`,
+        `select books.id, platforms.name as platform, books.reader_url as readerUrl, books.title, books.authors from books left join platforms on books.platform_id = platforms.id where books.id = ?`,
         id,
       );
 
+      const book: Book | undefined = row && {
+        ...row,
+        authors: JSON.parse(row.authors),
+      };
+
       return book;
     },
     async getBooks(): Promise<Array<Book>> {
-      const books: Array<Book> = await db.all(
+      const rows = await db.all(
-        `select books.id, platforms.name as platform, books.reader_url as readerUrl from books left join platforms on books.platform_id = platforms.id`,
+        `select books.id, platforms.name as platform, books.reader_url as readerUrl, books.title, books.authors from books left join platforms on books.platform_id = platforms.id`,
       );
 
+      const books: Array<Book> = rows.map((row) => ({
+        ...row,
+        authors: JSON.parse(row.authors),
+      }));
+
       return books;
     },
     async archive(dir: string) {

main.ts

@@ -61,7 +61,12 @@ const options = {
       const db = await createDatabase(args.values.db!);
       const library = createLibrary(db);
       const books = await library.getBooks();
-      console.dir(books, { depth: null });
+
+      console.dir(books, {
+        depth: null,
+        maxArrayLength: null,
+        maxStringLength: null,
+      });
     },
   },
   view: {
@@ -75,7 +80,24 @@ const options = {
         process.exit(1);
       }
 
-      console.dir(book, { depth: null });
+      console.dir(book, {
+        depth: null,
+        maxArrayLength: null,
+        maxStringLength: null,
+      });
+    },
+  },
+  pull: {
+    type: "boolean",
+    async run() {
+      const db = await createDatabase(args.values.db!);
+      const library = createLibrary(db);
+      const browser = await chromium.launch();
+      const platform = createPlatform({ db, browser });
+
+      for await (const book of platform.pull()) {
+        await library.add(book);
+      }
     },
   },
   download: {

migrations/2_add_tile_authors.sql

@@ -0,0 +1,2 @@
+alter table books add column title    text not null default '';
+alter table books add column authors  json not null default '[]';

platforms/dmm-books.ts

@@ -132,6 +132,105 @@ export function DmmBooks({ db, browser }: { db: Database; browser: Browser }) {
     return ctx;
   }
 
+  async function* getSeriesBooks(
+    ctx: BrowserContext,
+    series: {
+      seriesId: string;
+      shopName: string;
+      title: string;
+      authors: Array<string>;
+    },
+  ): AsyncGenerator<Book> {
+    const endpoint = `https://book.dmm.com/ajax/bff/contents/?shop_name=${series.shopName}&series_id=${series.seriesId}`;
+    const pager = {
+      page: 1,
+      perPage: 0,
+      totalCount: Infinity,
+    };
+
+    while (pager.page * pager.perPage <= pager.totalCount) {
+      const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
+
+      if (!res.ok()) {
+        throw new Error(`${res.status()} ${res.statusText()}`);
+      }
+
+      const body: {
+        volume_books: Array<{
+          title: string;
+          purchased?: {
+            streaming_url: string;
+          };
+        }>;
+        pager: {
+          page: number;
+          per_page: number;
+          total_count: number;
+        };
+      } = await res.json();
+
+      for (const book of body.volume_books.filter((b) => b.purchased)) {
+        yield {
+          id: NaN,
+          platform: "dmm-books",
+          readerUrl: book.purchased?.streaming_url!,
+          title: book.title || series.title || "",
+          authors: series.authors,
+        };
+
+        process.stderr.write(".");
+      }
+
+      pager.page += 1;
+      pager.perPage = body.pager.per_page;
+      pager.totalCount = body.pager.total_count;
+    }
+  }
+
+  async function* getAllBooks(ctx: BrowserContext): AsyncGenerator<Book> {
+    const endpoint = "https://book.dmm.com/ajax/bff/library/?shop_name=all";
+    const pager = {
+      page: 1,
+      perPage: 0,
+      totalCount: Infinity,
+    };
+
+    while (pager.page * pager.perPage <= pager.totalCount) {
+      const res = await ctx.request.get(`${endpoint}&page=${pager.page}`);
+
+      if (!res.ok()) {
+        throw new Error(`${res.status()} ${res.statusText()}`);
+      }
+
+      const body: {
+        series_books: Array<{
+          shop_name: string;
+          series_id: string;
+          title: string;
+          author: Array<string>;
+        }>;
+        pager: {
+          page: number;
+          per_page: number;
+          total_count: number;
+        };
+      } = await res.json();
+
+      for (const series of body.series_books) {
+        yield* getSeriesBooks(ctx, {
+          seriesId: series.series_id,
+          shopName: series.shop_name,
+          title: series.title,
+          authors: series.author,
+        });
+      }
+
+      pager.page += 1;
+      pager.perPage = body.pager.per_page;
+      pager.totalCount = body.pager.total_count;
+    }
+  }
+
   return {
     async login() {
       const ctx = await browser.newContext();
@@ -145,12 +244,24 @@ export function DmmBooks({ db, browser }: { db: Database; browser: Browser }) {
         JSON.stringify(secrets),
       );
     },
+
     async logout() {
       await browser.close();
       await db.run(
         `update platforms set secrets = 'null' where name = 'dmm-books'`,
       );
     },
+
+    async *pull(): AsyncGenerator<Book> {
+      const ctx = await loadBrowserContext();
+
+      yield* getAllBooks(ctx);
+
+      process.stderr.write(`\n`);
+
+      await browser.close();
+    },
+
     async download(dir: string, book: Book) {
       const ctx = await loadBrowserContext();
       const page = await ctx.newPage();