// Copyright 2010-2020, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "dictionary/system/system_dictionary.h"
#include <algorithm>
#include <cstdlib>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "base/file_util.h"
#include "base/logging.h"
#include "base/port.h"
#include "base/stl_util.h"
#include "base/system_util.h"
#include "base/util.h"
#include "config/config_handler.h"
#include "data_manager/testing/mock_data_manager.h"
#include "dictionary/dictionary_test_util.h"
#include "dictionary/dictionary_token.h"
#include "dictionary/pos_matcher.h"
#include "dictionary/system/codec_interface.h"
#include "dictionary/system/system_dictionary_builder.h"
#include "dictionary/text_dictionary_loader.h"
#include "protocol/commands.pb.h"
#include "protocol/config.pb.h"
#include "request/conversion_request.h"
#include "testing/base/public/googletest.h"
#include "testing/base/public/gunit.h"
#include "testing/base/public/mozctest.h"
#include "absl/strings/string_view.h"
DEFINE_int32(dictionary_test_size, 100000, "Dictionary size for this test.");
DEFINE_int32(dictionary_reverse_lookup_test_size, 1000,
"Number of tokens to run reverse lookup test.");
DECLARE_int32(min_key_length_to_use_small_cost_encoding);
namespace mozc {
namespace dictionary {
class SystemDictionaryTest : public ::testing::Test {
protected:
SystemDictionaryTest()
: pos_matcher_(mock_data_manager_.GetPOSMatcherData()),
text_dict_(new TextDictionaryLoader(pos_matcher_)),
dic_fn_(FileUtil::JoinPath(FLAGS_test_tmpdir, "mozc.dic")) {
const std::string dic_path = mozc::testing::GetSourceFileOrDie(
{"data", "dictionary_oss", "dictionary00.txt"});
text_dict_->LoadWithLineLimit(dic_path, "", FLAGS_dictionary_test_size);
convreq_.set_request(&request_);
convreq_.set_config(&config_);
}
void SetUp() override {
// Don't use small cost encoding by default.
original_flags_min_key_length_to_use_small_cost_encoding_ =
FLAGS_min_key_length_to_use_small_cost_encoding;
FLAGS_min_key_length_to_use_small_cost_encoding =
std::numeric_limits<int32>::max();
request_.Clear();
config::ConfigHandler::GetDefaultConfig(&config_);
}
void TearDown() override {
FLAGS_min_key_length_to_use_small_cost_encoding =
original_flags_min_key_length_to_use_small_cost_encoding_;
// This config initialization will be removed once ConversionRequest can
// take config as an injected argument.
config::Config config;
config::ConfigHandler::GetDefaultConfig(&config);
config::ConfigHandler::SetConfig(config);
}
void BuildSystemDictionary(const std::vector<Token *> &tokens,
int num_tokens);
Token *CreateToken(const std::string &key, const std::string &value) const;
bool CompareTokensForLookup(const Token &a, const Token &b,
bool reverse) const;
const testing::ScopedTmpUserProfileDirectory scoped_profile_dir_;
const testing::MockDataManager mock_data_manager_;
dictionary::POSMatcher pos_matcher_;
std::unique_ptr<TextDictionaryLoader> text_dict_;
ConversionRequest convreq_;
config::Config config_;
commands::Request request_;
const std::string dic_fn_;
int original_flags_min_key_length_to_use_small_cost_encoding_;
};
void SystemDictionaryTest::BuildSystemDictionary(
const std::vector<Token *> &source, int num_tokens) {
SystemDictionaryBuilder builder;
std::vector<Token *> tokens;
// Picks up first tokens.
for (std::vector<Token *>::const_iterator it = source.begin();
tokens.size() < num_tokens && it != source.end(); ++it) {
tokens.push_back(*it);
}
builder.BuildFromTokens(tokens);
builder.WriteToFile(dic_fn_);
}
Token *SystemDictionaryTest::CreateToken(const std::string &key,
const std::string &value) const {
Token *t = new Token;
t->key = key;
t->value = value;
t->cost = 0;
t->lid = 0;
t->rid = 0;
return t;
}
// Returns true if they seem to be same
bool SystemDictionaryTest::CompareTokensForLookup(const Token &a,
const Token &b,
bool reverse) const {
const bool key_value_check = reverse ? (a.key == b.value && a.value == b.key)
: (a.key == b.key && a.value == b.value);
if (!key_value_check) {
return false;
}
const bool comp_cost = a.cost == b.cost;
if (!comp_cost) {
return false;
}
const bool spelling_match = (a.attributes & Token::SPELLING_CORRECTION) ==
(b.attributes & Token::SPELLING_CORRECTION);
if (!spelling_match) {
return false;
}
const bool id_match = (a.lid == b.lid) && (a.rid == b.rid);
if (!id_match) {
return false;
}
return true;
}
TEST_F(SystemDictionaryTest, HasValue) {
std::vector<Token *> tokens;
for (int i = 0; i < 4; ++i) {
Token *token = new Token;
token->key = Util::StringPrintf("きー%d", i);
token->value = Util::StringPrintf("バリュー%d", i);
tokens.push_back(token);
}
{ // Alphabet
Token *token = new Token;
token->key = "Mozc";
token->value = "Mozc";
tokens.push_back(token);
}
{ // Alphabet upper case
Token *token = new Token;
token->key = "upper";
token->value = "UPPER";
tokens.push_back(token);
}
const std::string kFull = "ｆｕｌｌ";
const std::string kHiragana = "ひらがな";
const std::string kKatakanaKey = "かたかな";
const std::string kKatakanaValue = "カタカナ";
{ // Alphabet full width
Token *token = new Token;
token->key = "full";
token->value = kFull;
tokens.push_back(token);
}
{ // Hiragana
Token *token = new Token;
token->key = kHiragana;
token->value = kHiragana;
tokens.push_back(token);
}
{ // Katakana
Token *token = new Token;
token->key = kKatakanaKey;
token->value = kKatakanaValue;
tokens.push_back(token);
}
BuildSystemDictionary(tokens, tokens.size());
auto system_dic = SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic.get() != nullptr)
<< "Failed to open dictionary source:" << dic_fn_;
EXPECT_TRUE(system_dic->HasValue("バリュー0"));
EXPECT_TRUE(system_dic->HasValue("バリュー1"));
EXPECT_TRUE(system_dic->HasValue("バリュー2"));
EXPECT_TRUE(system_dic->HasValue("バリュー3"));
EXPECT_FALSE(system_dic->HasValue("バリュー4"));
EXPECT_FALSE(system_dic->HasValue("バリュー5"));
EXPECT_FALSE(system_dic->HasValue("バリュー6"));
EXPECT_TRUE(system_dic->HasValue("Mozc"));
EXPECT_FALSE(system_dic->HasValue("mozc"));
EXPECT_TRUE(system_dic->HasValue("UPPER"));
EXPECT_FALSE(system_dic->HasValue("upper"));
EXPECT_TRUE(system_dic->HasValue(kFull));
EXPECT_FALSE(system_dic->HasValue("full"));
EXPECT_TRUE(system_dic->HasValue(kHiragana));
EXPECT_FALSE(system_dic->HasValue("ヒラガナ\n"));
EXPECT_TRUE(system_dic->HasValue(kKatakanaValue));
EXPECT_FALSE(system_dic->HasValue(kKatakanaKey));
STLDeleteElements(&tokens);
}
TEST_F(SystemDictionaryTest, NormalWord) {
std::vector<Token *> source_tokens;
std::unique_ptr<Token> t0(new Token);
t0->key = "あ";
t0->value = "亜";
t0->cost = 100;
t0->lid = 50;
t0->rid = 70;
source_tokens.push_back(t0.get());
BuildSystemDictionary(source_tokens, FLAGS_dictionary_test_size);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
CollectTokenCallback callback;
// Look up by exact key.
system_dic->LookupPrefix(t0->key, convreq_, &callback);
ASSERT_EQ(1, callback.tokens().size());
EXPECT_TOKEN_EQ(*t0, callback.tokens().front());
// Look up by prefix.
callback.Clear();
system_dic->LookupPrefix("あいう", convreq_, &callback);
ASSERT_EQ(1, callback.tokens().size());
EXPECT_TOKEN_EQ(*t0, callback.tokens().front());
// Nothing should be looked up.
callback.Clear();
system_dic->LookupPrefix("かきく", convreq_, &callback);
EXPECT_TRUE(callback.tokens().empty());
}
TEST_F(SystemDictionaryTest, SameWord) {
std::vector<Token> tokens(4);
tokens[0].key = "あ";
tokens[0].value = "亜";
tokens[0].cost = 100;
tokens[0].lid = 50;
tokens[0].rid = 70;
tokens[1].key = "あ";
tokens[1].value = "亜";
tokens[1].cost = 150;
tokens[1].lid = 100;
tokens[1].rid = 200;
tokens[2].key = "あ";
tokens[2].value = "あ";
tokens[2].cost = 100;
tokens[2].lid = 1000;
tokens[2].rid = 2000;
tokens[3].key = "あ";
tokens[3].value = "亜";
tokens[3].cost = 1000;
tokens[3].lid = 2000;
tokens[3].rid = 3000;
std::vector<Token *> source_tokens;
for (size_t i = 0; i < tokens.size(); ++i) {
source_tokens.push_back(&tokens[i]);
}
BuildSystemDictionary(source_tokens, FLAGS_dictionary_test_size);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
// All the tokens should be looked up.
CollectTokenCallback callback;
system_dic->LookupPrefix("あ", convreq_, &callback);
EXPECT_TOKENS_EQ_UNORDERED(source_tokens, callback.tokens());
}
TEST_F(SystemDictionaryTest, LookupAllWords) {
const std::vector<Token *> &source_tokens = text_dict_->tokens();
BuildSystemDictionary(source_tokens, FLAGS_dictionary_test_size);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
// All the tokens should be looked up.
for (size_t i = 0; i < source_tokens.size(); ++i) {
CheckTokenExistenceCallback callback(source_tokens[i]);
system_dic->LookupPrefix(source_tokens[i]->key, convreq_, &callback);
EXPECT_TRUE(callback.found())
<< "Token was not found: " << PrintToken(*source_tokens[i]);
}
}
TEST_F(SystemDictionaryTest, SimpleLookupPrefix) {
const std::string k0 = "は";
const std::string k1 = "はひふへほ";
std::unique_ptr<Token> t0(CreateToken(k0, "aa"));
std::unique_ptr<Token> t1(CreateToken(k1, "bb"));
std::vector<Token *> source_tokens;
source_tokens.push_back(t0.get());
source_tokens.push_back(t1.get());
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, 100);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
// |t0| should be looked up from |k1|.
CheckTokenExistenceCallback callback(t0.get());
system_dic->LookupPrefix(k1, convreq_, &callback);
EXPECT_TRUE(callback.found());
}
namespace {
class LookupPrefixTestCallback : public SystemDictionary::Callback {
public:
ResultType OnKey(absl::string_view key) override {
if (key == "かき") {
return TRAVERSE_CULL;
} else if (key == "さ") {
return TRAVERSE_NEXT_KEY;
} else if (key == "た") {
return TRAVERSE_DONE;
}
return TRAVERSE_CONTINUE;
}
ResultType OnToken(absl::string_view key, absl::string_view actual_key,
const Token &token) override {
result_.insert(std::make_pair(token.key, token.value));
return TRAVERSE_CONTINUE;
}
const std::set<std::pair<std::string, std::string>> &result() const {
return result_;
}
private:
std::set<std::pair<std::string, std::string>> result_;
};
} // namespace
TEST_F(SystemDictionaryTest, LookupPrefix) {
// Set up a test dictionary.
struct {
const char *key;
const char *value;
} kKeyValues[] = {
{"あ", "亜"}, {"あ", "安"}, {"あ", "在"},
{"あい", "愛"}, {"あい", "藍"}, {"あいう", "藍雨"},
{"か", "可"}, {"かき", "牡蠣"}, {"かき", "夏季"},
{"かきく", "柿久"}, {"さ", "差"}, {"さ", "左"},
{"さし", "刺"}, {"た", "田"}, {"た", "多"},
{"たち", "多値"}, {"たちつ", "タチツ"}, {"は", "葉"},
{"は", "歯"}, {"はひ", "ハヒ"}, {"ば", "場"},
{"はび", "波美"}, {"ばび", "馬尾"}, {"ばびぶ", "バビブ"},
};
const size_t kKeyValuesSize = arraysize(kKeyValues);
std::unique_ptr<Token> tokens[kKeyValuesSize];
std::vector<Token *> source_tokens(kKeyValuesSize);
for (size_t i = 0; i < kKeyValuesSize; ++i) {
tokens[i].reset(CreateToken(kKeyValues[i].key, kKeyValues[i].value));
source_tokens[i] = tokens[i].get();
}
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, kKeyValuesSize);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
// Test for normal prefix lookup without key expansion.
{
LookupPrefixTestCallback callback;
system_dic->LookupPrefix("あい", // "あい"
convreq_, &callback);
const std::set<std::pair<std::string, std::string>> &result =
callback.result();
// "あ" -- "あい" should be found.
for (size_t i = 0; i < 5; ++i) {
const std::pair<std::string, std::string> entry(kKeyValues[i].key,
kKeyValues[i].value);
EXPECT_TRUE(result.end() != result.find(entry));
}
// The others should not be found.
for (size_t i = 5; i < arraysize(kKeyValues); ++i) {
const std::pair<std::string, std::string> entry(kKeyValues[i].key,
kKeyValues[i].value);
EXPECT_TRUE(result.end() == result.find(entry));
}
}
// Test for normal prefix lookup without key expansion, but with culling
// feature.
{
LookupPrefixTestCallback callback;
system_dic->LookupPrefix("かきく", convreq_, &callback);
const std::set<std::pair<std::string, std::string>> &result =
callback.result();
// Only "か" should be found as the callback doesn't traverse the subtree of
// "かき" due to culling request from LookupPrefixTestCallback::OnKey().
for (size_t i = 0; i < kKeyValuesSize; ++i) {
const std::pair<std::string, std::string> entry(kKeyValues[i].key,
kKeyValues[i].value);
EXPECT_EQ(entry.first == "か", result.find(entry) != result.end());
}
}
// Test for TRAVERSE_NEXT_KEY.
{
LookupPrefixTestCallback callback;
system_dic->LookupPrefix("さしす", convreq_, &callback);
const std::set<std::pair<std::string, std::string>> &result =
callback.result();
// Only "さし" should be found as tokens for "さ" is skipped (see
// LookupPrefixTestCallback::OnKey()).
for (size_t i = 0; i < kKeyValuesSize; ++i) {
const std::pair<std::string, std::string> entry(kKeyValues[i].key,
kKeyValues[i].value);
EXPECT_EQ(entry.first == "さし", result.find(entry) != result.end());
}
}
// Test for TRAVERSE_DONE.
{
LookupPrefixTestCallback callback;
system_dic->LookupPrefix("たちつ", convreq_, &callback);
const std::set<std::pair<std::string, std::string>> &result =
callback.result();
// Nothing should be found as the traversal is immediately done after seeing
// "た"; see LookupPrefixTestCallback::OnKey().
EXPECT_TRUE(result.empty());
}
// Test for prefix lookup with key expansion.
{
LookupPrefixTestCallback callback;
// Use kana modifier insensitive lookup
request_.set_kana_modifier_insensitive_conversion(true);
config_.set_use_kana_modifier_insensitive_conversion(true);
system_dic->LookupPrefix("はひ", convreq_, &callback);
const std::set<std::pair<std::string, std::string>> &result =
callback.result();
const char *kExpectedKeys[] = {
"は", "ば", "はひ", "ばひ", "はび", "ばび",
};
const std::set<std::string> expected(
kExpectedKeys, kExpectedKeys + arraysize(kExpectedKeys));
for (size_t i = 0; i < kKeyValuesSize; ++i) {
const bool to_be_found =
expected.find(kKeyValues[i].key) != expected.end();
const std::pair<std::string, std::string> entry(kKeyValues[i].key,
kKeyValues[i].value);
EXPECT_EQ(to_be_found, result.find(entry) != result.end());
}
}
}
TEST_F(SystemDictionaryTest, LookupPredictive) {
std::vector<Token *> tokens;
ScopedElementsDeleter<std::vector<Token *>> deleter(&tokens);
tokens.push_back(CreateToken("まみむめもや", "value0"));
tokens.push_back(CreateToken("まみむめもやゆよ", "value1"));
// Build a dictionary with the above two tokens plus those from test data.
{
std::vector<Token *> source_tokens = tokens;
text_dict_->CollectTokens(&source_tokens); // Load test data.
BuildSystemDictionary(source_tokens, 10000);
}
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source: " << dic_fn_;
// All the tokens in |tokens| should be looked up by "まみむめも".
const char kMamimumemo[] = "まみむめも";
CheckMultiTokensExistenceCallback callback(tokens);
system_dic->LookupPredictive(kMamimumemo, convreq_, &callback);
EXPECT_TRUE(callback.AreAllFound());
}
TEST_F(SystemDictionaryTest, LookupPredictive_KanaModifierInsensitiveLookup) {
std::vector<Token *> tokens;
ScopedElementsDeleter<std::vector<Token *>> deleter(&tokens);
tokens.push_back(CreateToken("がっこう", "学校"));
tokens.push_back(CreateToken("かっこう", "格好"));
BuildSystemDictionary(tokens, 100);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic.get() != NULL)
<< "Failed to open dictionary source: " << dic_fn_;
const std::string kKey = "かつこう";
// Without Kana modifier insensitive lookup flag, nothing is looked up.
CollectTokenCallback callback;
request_.set_kana_modifier_insensitive_conversion(false);
config_.set_use_kana_modifier_insensitive_conversion(false);
system_dic->LookupPredictive(kKey, convreq_, &callback);
EXPECT_TRUE(callback.tokens().empty());
// With Kana modifier insensitive lookup flag, every token is looked up.
callback.Clear();
request_.set_kana_modifier_insensitive_conversion(true);
config_.set_use_kana_modifier_insensitive_conversion(true);
system_dic->LookupPredictive(kKey, convreq_, &callback);
EXPECT_TOKENS_EQ_UNORDERED(tokens, callback.tokens());
}
TEST_F(SystemDictionaryTest, LookupPredictive_CutOffEmulatingBFS) {
std::vector<Token *> tokens;
ScopedElementsDeleter<std::vector<Token *>> deleter(&tokens);
tokens.push_back(CreateToken("あい", "ai"));
tokens.push_back(CreateToken("あいうえお", "aiueo"));
// Build a dictionary with the above two tokens plus those from test data.
{
std::vector<Token *> source_tokens = tokens;
text_dict_->CollectTokens(&source_tokens); // Load test data.
BuildSystemDictionary(source_tokens, 10000);
}
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source: " << dic_fn_;
// Since there are many entries starting with "あ" in test dictionary, it's
// expected that "あいうえお" is not looked up because of longer key cut-off
// mechanism. However, "あい" is looked up as it's short.
CheckMultiTokensExistenceCallback callback(tokens);
system_dic->LookupPredictive("あ", convreq_, &callback);
EXPECT_TRUE(callback.IsFound(tokens[0]));
EXPECT_FALSE(callback.IsFound(tokens[1]));
}
TEST_F(SystemDictionaryTest, LookupExact) {
std::vector<Token *> source_tokens;
const std::string k0 = "は";
const std::string k1 = "はひふへほ";
std::unique_ptr<Token> t0(CreateToken(k0, "aa"));
std::unique_ptr<Token> t1(CreateToken(k1, "bb"));
source_tokens.push_back(t0.get());
source_tokens.push_back(t1.get());
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, 100);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
// |t0| should not be looked up from |k1|.
CheckTokenExistenceCallback callback0(t0.get());
system_dic->LookupExact(k1, convreq_, &callback0);
EXPECT_FALSE(callback0.found());
// But |t1| should be found.
CheckTokenExistenceCallback callback1(t1.get());
system_dic->LookupExact(k1, convreq_, &callback1);
EXPECT_TRUE(callback1.found());
// Nothing should be found from "hoge".
CollectTokenCallback callback_hoge;
system_dic->LookupExact("hoge", convreq_, &callback_hoge);
EXPECT_TRUE(callback_hoge.tokens().empty());
}
TEST_F(SystemDictionaryTest, LookupReverse) {
std::unique_ptr<Token> t0(new Token);
t0->key = "ど";
t0->value = "ド";
t0->cost = 1;
t0->lid = 2;
t0->rid = 3;
std::unique_ptr<Token> t1(new Token);
t1->key = "どらえもん";
t1->value = "ドラえもん";
t1->cost = 1;
t1->lid = 2;
t1->rid = 3;
std::unique_ptr<Token> t2(new Token);
t2->key = "といざらす®";
t2->value = "トイザらス®";
t2->cost = 1;
t2->lid = 2;
t2->rid = 3;
std::unique_ptr<Token> t3(new Token);
// Both t3 and t4 will be encoded into 3 bytes.
t3->key = "ああああああ";
t3->value = t3->key;
t3->cost = 32000;
t3->lid = 1;
t3->rid = 1;
std::unique_ptr<Token> t4(new Token);
*t4 = *t3;
t4->lid = 1;
t4->rid = 2;
std::unique_ptr<Token> t5(new Token);
// t5 will be encoded into 3 bytes.
t5->key = "いいいいいい";
t5->value = t5->key;
t5->cost = 32000;
t5->lid = 1;
t5->rid = 1;
// spelling correction token should not be retrieved by reverse lookup.
std::unique_ptr<Token> t6(new Token);
t6->key = "どらえもん";
t6->value = "ドラえもん";
t6->cost = 1;
t6->lid = 2;
t6->rid = 3;
t6->attributes = Token::SPELLING_CORRECTION;
std::unique_ptr<Token> t7(new Token);
t7->key = "こんさーと";
t7->value = "コンサート";
t7->cost = 1;
t7->lid = 1;
t7->rid = 1;
// "バージョン" should not return a result with the key "ヴァージョン".
std::unique_ptr<Token> t8(new Token);
t8->key = "ばーじょん";
t8->value = "バージョン";
t8->cost = 1;
t8->lid = 1;
t8->rid = 1;
std::vector<Token *> source_tokens;
source_tokens.push_back(t0.get());
source_tokens.push_back(t1.get());
source_tokens.push_back(t2.get());
source_tokens.push_back(t3.get());
source_tokens.push_back(t4.get());
source_tokens.push_back(t5.get());
source_tokens.push_back(t6.get());
source_tokens.push_back(t7.get());
source_tokens.push_back(t8.get());
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, source_tokens.size());
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
const size_t test_size =
std::min(static_cast<size_t>(FLAGS_dictionary_reverse_lookup_test_size),
source_tokens.size());
for (size_t source_index = 0; source_index < test_size; ++source_index) {
const Token &source_token = *source_tokens[source_index];
CollectTokenCallback callback;
system_dic->LookupReverse(source_token.value, convreq_, &callback);
const std::vector<Token> &tokens = callback.tokens();
bool found = false;
for (size_t i = 0; i < tokens.size(); ++i) {
const Token &token = tokens[i];
// Make sure any of the key lengths of the lookup results
// doesn't exceed the original key length.
// It happened once
// when called with "バージョン", returning "ヴァージョン".
EXPECT_LE(token.key.size(), source_token.value.size())
<< token.key << ":" << token.value << "\t" << source_token.value;
if (CompareTokensForLookup(source_token, token, true)) {
found = true;
}
}
if ((source_token.attributes & Token::SPELLING_CORRECTION) ==
Token::SPELLING_CORRECTION) {
EXPECT_FALSE(found) << "Spelling correction token was retrieved:"
<< PrintToken(source_token);
if (found) {
return;
}
} else {
EXPECT_TRUE(found) << "Failed to find " << source_token.key << ":"
<< source_token.value;
if (!found) {
return;
}
}
}
{
// test for non exact transliterated index string.
// append "が"
const std::string key = t7->value + "が";
CollectTokenCallback callback;
system_dic->LookupReverse(key, convreq_, &callback);
const std::vector<Token> &tokens = callback.tokens();
bool found = false;
for (size_t i = 0; i < tokens.size(); ++i) {
if (CompareTokensForLookup(*t7, tokens[i], true)) {
found = true;
}
}
EXPECT_TRUE(found) << "Missed token for non exact transliterated index "
<< key;
}
}
TEST_F(SystemDictionaryTest, LookupReverseIndex) {
const std::vector<Token *> &source_tokens = text_dict_->tokens();
BuildSystemDictionary(source_tokens, FLAGS_dictionary_test_size);
std::unique_ptr<SystemDictionary> system_dic_without_index =
SystemDictionary::Builder(dic_fn_)
.SetOptions(SystemDictionary::NONE)
.Build()
.value();
ASSERT_TRUE(system_dic_without_index)
<< "Failed to open dictionary source:" << dic_fn_;
std::unique_ptr<SystemDictionary> system_dic_with_index =
SystemDictionary::Builder(dic_fn_)
.SetOptions(SystemDictionary::ENABLE_REVERSE_LOOKUP_INDEX)
.Build()
.value();
ASSERT_TRUE(system_dic_with_index)
<< "Failed to open dictionary source:" << dic_fn_;
std::vector<Token *>::const_iterator it;
int size = FLAGS_dictionary_reverse_lookup_test_size;
for (it = source_tokens.begin(); size > 0 && it != source_tokens.end();
++it, --size) {
const Token &t = **it;
CollectTokenCallback callback1, callback2;
system_dic_without_index->LookupReverse(t.value, convreq_, &callback1);
system_dic_with_index->LookupReverse(t.value, convreq_, &callback2);
const std::vector<Token> &tokens1 = callback1.tokens();
const std::vector<Token> &tokens2 = callback2.tokens();
ASSERT_EQ(tokens1.size(), tokens2.size());
for (size_t i = 0; i < tokens1.size(); ++i) {
EXPECT_TOKEN_EQ(tokens1[i], tokens2[i]);
}
}
}
TEST_F(SystemDictionaryTest, LookupReverseWithCache) {
const std::string kDoraemon = "ドラえもん";
Token source_token;
source_token.key = "どらえもん";
source_token.value = kDoraemon;
source_token.cost = 1;
source_token.lid = 2;
source_token.rid = 3;
std::vector<Token *> source_tokens;
source_tokens.push_back(&source_token);
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, source_tokens.size());
Token target_token = source_token;
target_token.key.swap(target_token.value);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
system_dic->PopulateReverseLookupCache(kDoraemon);
CheckTokenExistenceCallback callback(&target_token);
system_dic->LookupReverse(kDoraemon, convreq_, &callback);
EXPECT_TRUE(callback.found())
<< "Could not find " << PrintToken(source_token);
system_dic->ClearReverseLookupCache();
}
TEST_F(SystemDictionaryTest, SpellingCorrectionTokens) {
std::vector<Token> tokens(3);
tokens[0].key = "あぼがど";
tokens[0].value = "アボカド";
tokens[0].cost = 1;
tokens[0].lid = 0;
tokens[0].rid = 2;
tokens[0].attributes = Token::SPELLING_CORRECTION;
tokens[1].key = "しゅみれーしょん";
tokens[1].value = "シミュレーション";
tokens[1].cost = 1;
tokens[1].lid = 100;
tokens[1].rid = 3;
tokens[1].attributes = Token::SPELLING_CORRECTION;
tokens[2].key = "あきはばら";
tokens[2].value = "秋葉原";
tokens[2].cost = 1000;
tokens[2].lid = 1;
tokens[2].rid = 2;
std::vector<Token *> source_tokens;
for (size_t i = 0; i < tokens.size(); ++i) {
source_tokens.push_back(&tokens[i]);
}
BuildSystemDictionary(source_tokens, source_tokens.size());
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
for (size_t i = 0; i < source_tokens.size(); ++i) {
CheckTokenExistenceCallback callback(source_tokens[i]);
system_dic->LookupPrefix(source_tokens[i]->key, convreq_, &callback);
EXPECT_TRUE(callback.found())
<< "Token " << i << " was not found: " << PrintToken(*source_tokens[i]);
}
}
TEST_F(SystemDictionaryTest, EnableNoModifierTargetWithLoudsTrie) {
const std::string k0 = "かつ";
const std::string k1 = "かっこ";
const std::string k2 = "かつこう";
const std::string k3 = "かっこう";
const std::string k4 = "がっこう";
std::unique_ptr<Token> tokens[5];
tokens[0].reset(CreateToken(k0, "aa"));
tokens[1].reset(CreateToken(k1, "bb"));
tokens[2].reset(CreateToken(k2, "cc"));
tokens[3].reset(CreateToken(k3, "dd"));
tokens[4].reset(CreateToken(k4, "ee"));
std::vector<Token *> source_tokens;
for (size_t i = 0; i < arraysize(tokens); ++i) {
source_tokens.push_back(tokens[i].get());
}
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, 100);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
request_.set_kana_modifier_insensitive_conversion(true);
config_.set_use_kana_modifier_insensitive_conversion(true);
// Prefix search
for (size_t i = 0; i < arraysize(tokens); ++i) {
CheckTokenExistenceCallback callback(tokens[i].get());
// "かつこう" -> "かつ", "かっこ", "かつこう", "かっこう" and "がっこう"
system_dic->LookupPrefix(k2, convreq_, &callback);
EXPECT_TRUE(callback.found())
<< "Token " << i << " was not found: " << PrintToken(*tokens[i]);
}
// Predictive searches
{
// "かつ" -> "かつ", "かっこ", "かつこう", "かっこう" and "がっこう"
std::vector<Token *> expected;
for (size_t i = 0; i < arraysize(tokens); ++i) {
expected.push_back(tokens[i].get());
}
CheckMultiTokensExistenceCallback callback(expected);
system_dic->LookupPredictive(k0, convreq_, &callback);
EXPECT_TRUE(callback.AreAllFound());
}
{
// "かっこ" -> "かっこ", "かっこう" and "がっこう"
std::vector<Token *> expected;
expected.push_back(tokens[1].get());
expected.push_back(tokens[3].get());
expected.push_back(tokens[4].get());
CheckMultiTokensExistenceCallback callback(expected);
system_dic->LookupPredictive(k1, convreq_, &callback);
EXPECT_TRUE(callback.AreAllFound());
}
}
TEST_F(SystemDictionaryTest, NoModifierForKanaEntries) {
std::unique_ptr<Token> t0(CreateToken("ていすてぃんぐ", "テイスティング"));
std::unique_ptr<Token> t1(CreateToken("てすとです", "てすとです"));
std::vector<Token *> source_tokens;
source_tokens.push_back(t0.get());
source_tokens.push_back(t1.get());
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, 100);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
// Lookup |t0| from "ていすていんぐ"
const std::string k = "ていすていんぐ";
request_.set_kana_modifier_insensitive_conversion(true);
config_.set_use_kana_modifier_insensitive_conversion(true);
CheckTokenExistenceCallback callback(t0.get());
system_dic->LookupPrefix(k, convreq_, &callback);
EXPECT_TRUE(callback.found()) << "Not found: " << PrintToken(*t0);
}
TEST_F(SystemDictionaryTest, DoNotReturnNoModifierTargetWithLoudsTrie) {
const std::string k0 = "かつ";
const std::string k1 = "かっこ";
const std::string k2 = "かつこう";
const std::string k3 = "かっこう";
const std::string k4 = "がっこう";
std::unique_ptr<Token> t0(CreateToken(k0, "aa"));
std::unique_ptr<Token> t1(CreateToken(k1, "bb"));
std::unique_ptr<Token> t2(CreateToken(k2, "cc"));
std::unique_ptr<Token> t3(CreateToken(k3, "dd"));
std::unique_ptr<Token> t4(CreateToken(k4, "ee"));
std::vector<Token *> source_tokens;
source_tokens.push_back(t0.get());
source_tokens.push_back(t1.get());
source_tokens.push_back(t2.get());
source_tokens.push_back(t3.get());
source_tokens.push_back(t4.get());
text_dict_->CollectTokens(&source_tokens);
BuildSystemDictionary(source_tokens, 100);
std::unique_ptr<SystemDictionary> system_dic =
SystemDictionary::Builder(dic_fn_).Build().value();
ASSERT_TRUE(system_dic) << "Failed to open dictionary source:" << dic_fn_;
request_.set_kana_modifier_insensitive_conversion(false);
config_.set_use_kana_modifier_insensitive_conversion(false);
// Prefix search
{
// "かっこう" (k3) -> "かっこ" (k1) and "かっこう" (k3)
// Make sure "がっこう" is not in the results when searched by "かっこう"
std::vector<Token *> to_be_looked_up, not_to_be_looked_up;
to_be_looked_up.push_back(t1.get());
to_be_looked_up.push_back(t3.get());
not_to_be_looked_up.push_back(t0.get());
not_to_be_looked_up.push_back(t2.get());
not_to_be_looked_up.push_back(t4.get());
for (size_t i = 0; i < to_be_looked_up.size(); ++i) {
CheckTokenExistenceCallback callback(to_be_looked_up[i]);
system_dic->LookupPrefix(k3, convreq_, &callback);
EXPECT_TRUE(callback.found())
<< "Token is not found: " << PrintToken(*to_be_looked_up[i]);
}
for (size_t i = 0; i < not_to_be_looked_up.size(); ++i) {
CheckTokenExistenceCallback callback(not_to_be_looked_up[i]);
system_dic->LookupPrefix(k3, convreq_, &callback);
EXPECT_FALSE(callback.found()) << "Token should not be found: "
<< PrintToken(*not_to_be_looked_up[i]);
}
}
// Predictive search
{
// "かっこ" -> "かっこ" and "かっこう"
// Make sure "がっこう" is not in the results when searched by "かっこ"
std::vector<Token *> to_be_looked_up, not_to_be_looked_up;
to_be_looked_up.push_back(t1.get());
to_be_looked_up.push_back(t3.get());
not_to_be_looked_up.push_back(t0.get());
not_to_be_looked_up.push_back(t2.get());
not_to_be_looked_up.push_back(t4.get());
for (size_t i = 0; i < to_be_looked_up.size(); ++i) {
CheckTokenExistenceCallback callback(to_be_looked_up[i]);
system_dic->LookupPredictive(k1, convreq_, &callback);
EXPECT_TRUE(callback.found())
<< "Token is not found: " << PrintToken(*to_be_looked_up[i]);
}
for (size_t i = 0; i < not_to_be_looked_up.size(); ++i) {
CheckTokenExistenceCallback callback(not_to_be_looked_up[i]);
system_dic->LookupPredictive(k3, convreq_, &callback);
EXPECT_FALSE(callback.found()) << "Token should not be found: "
<< PrintToken(*not_to_be_looked_up[i]);
}
}
}
} // namespace dictionary
} // namespace mozc