daraz-san/scripts/util/morpheme.js

const kuromoji = require("kuromoji");

const dicPath = require("path").resolve(
  require.resolve("kuromoji"),
  "../../dict"
);

const useTokenize = () =>
  new Promise((resolve, reject) =>
    kuromoji
      .builder({ dicPath })
      .build((err, tokenizer) =>
        err ? reject(err) : resolve(tokenizer.tokenize.bind(tokenizer))
      )
  );

const features = new Map([
  ["surface_form", "表層形"],
  ["pos", "品詞"],
  ["pos_detail_1", "品詞細分類1"],
  ["pos_detail_2", "品詞細分類2"],
  ["pos_detail_3", "品詞細分類3"],
  ["conjugated_type", "活用型"],
  ["conjugated_form", "活用形"],
  ["basic_form", "基本形"],
  ["reading", "読み"],
  ["pronunciation", "発音"],
]);

const toCSV = (tokens) =>
  [
    [...features.values()].join(","),
    ...tokens.map((token) =>
      [...features.keys()].map((feature) => token[feature]).join(",")
    ),
  ].join("\n");

module.exports = { useTokenize, toCSV };