mirror of
https://github.com/kou029w/daraz-san.git
synced 2025-01-18 16:08:06 +00:00
38 lines
948 B
JavaScript
38 lines
948 B
JavaScript
const kuromoji = require("kuromoji");
|
|
|
|
const dicPath = require("path").resolve(
|
|
require.resolve("kuromoji"),
|
|
"../../dict"
|
|
);
|
|
|
|
const useTokenize = () =>
|
|
new Promise((resolve, reject) =>
|
|
kuromoji
|
|
.builder({ dicPath })
|
|
.build((err, tokenizer) =>
|
|
err ? reject(err) : resolve(tokenizer.tokenize.bind(tokenizer))
|
|
)
|
|
);
|
|
|
|
const features = new Map([
|
|
["surface_form", "表層形"],
|
|
["pos", "品詞"],
|
|
["pos_detail_1", "品詞細分類1"],
|
|
["pos_detail_2", "品詞細分類2"],
|
|
["pos_detail_3", "品詞細分類3"],
|
|
["conjugated_type", "活用型"],
|
|
["conjugated_form", "活用形"],
|
|
["basic_form", "基本形"],
|
|
["reading", "読み"],
|
|
["pronunciation", "発音"],
|
|
]);
|
|
|
|
const toCSV = (tokens) =>
|
|
[
|
|
[...features.values()].join(","),
|
|
...tokens.map((token) =>
|
|
[...features.keys()].map((feature) => token[feature]).join(",")
|
|
),
|
|
].join("\n");
|
|
|
|
module.exports = { useTokenize, toCSV };
|