Allow keywords to match either substrings or whole words.
Word-boundary matching only works as intended in English and languages that use similar word-breaking characters; it doesn't work so well in (say) Japanese, Chinese, or Thai. It's unacceptable to have a feature that doesn't work as intended for some languages. (Moreso especially considering that it's likely that the largest contingent on the Mastodon bit of the fediverse speaks Japanese.) There are rules specified in Unicode TR29[1] for word-breaking across all languages supported by Unicode, but the rules deliberately do not cover all cases. In fact, TR29 states For example, reliable detection of word boundaries in languages such as Thai, Lao, Chinese, or Japanese requires the use of dictionary lookup, analogous to English hyphenation. So we aren't going to be able to make word detection work with regexes within Mastodon (or glitchsoc). However, for a first pass (even if it's kind of punting) we can allow the user to choose whether they want word or substring detection and warn about the limitations of this implementation in, say, docs. [1]: https://unicode.org/reports/tr29/ https://web.archive.org/web/20171001005125/https://unicode.org/reports/tr29/
This commit is contained in:
parent
2e03a10059
commit
4a64181461
|
@ -6,6 +6,7 @@
|
||||||
# id :integer not null, primary key
|
# id :integer not null, primary key
|
||||||
# account_id :integer not null
|
# account_id :integer not null
|
||||||
# keyword :string not null
|
# keyword :string not null
|
||||||
|
# whole_word :boolean default(TRUE), not null
|
||||||
# created_at :datetime not null
|
# created_at :datetime not null
|
||||||
# updated_at :datetime not null
|
# updated_at :datetime not null
|
||||||
#
|
#
|
||||||
|
@ -32,12 +33,13 @@ class KeywordMute < ApplicationRecord
|
||||||
|
|
||||||
def initialize(account_id)
|
def initialize(account_id)
|
||||||
re = [].tap do |arr|
|
re = [].tap do |arr|
|
||||||
KeywordMute.where(account_id: account_id).select(:keyword, :id).find_each do |m|
|
KeywordMute.where(account_id: account_id).select(:keyword, :id, :whole_word).find_each do |m|
|
||||||
arr << Regexp.escape(m.keyword.strip)
|
boundary = m.whole_word ? '\b' : ''
|
||||||
|
arr << "#{boundary}#{Regexp.escape(m.keyword.strip)}#{boundary}"
|
||||||
end
|
end
|
||||||
end.join('|')
|
end.join('|')
|
||||||
|
|
||||||
@regex = /\b(?:#{re})\b/i unless re.empty?
|
@regex = /#{re}/i unless re.empty?
|
||||||
end
|
end
|
||||||
|
|
||||||
def =~(str)
|
def =~(str)
|
||||||
|
|
|
@ -3,6 +3,7 @@ class CreateKeywordMutes < ActiveRecord::Migration[5.1]
|
||||||
create_table :keyword_mutes do |t|
|
create_table :keyword_mutes do |t|
|
||||||
t.references :account, null: false
|
t.references :account, null: false
|
||||||
t.string :keyword, null: false
|
t.string :keyword, null: false
|
||||||
|
t.boolean :whole_word, null: false, default: true
|
||||||
t.timestamps
|
t.timestamps
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -170,6 +170,7 @@ ActiveRecord::Schema.define(version: 20171010025614) do
|
||||||
create_table "keyword_mutes", force: :cascade do |t|
|
create_table "keyword_mutes", force: :cascade do |t|
|
||||||
t.bigint "account_id", null: false
|
t.bigint "account_id", null: false
|
||||||
t.string "keyword", null: false
|
t.string "keyword", null: false
|
||||||
|
t.boolean "whole_word", default: true, null: false
|
||||||
t.datetime "created_at", null: false
|
t.datetime "created_at", null: false
|
||||||
t.datetime "updated_at", null: false
|
t.datetime "updated_at", null: false
|
||||||
t.index ["account_id"], name: "index_keyword_mutes_on_account_id"
|
t.index ["account_id"], name: "index_keyword_mutes_on_account_id"
|
||||||
|
|
|
@ -30,10 +30,16 @@ RSpec.describe KeywordMute, type: :model do
|
||||||
expect(matcher =~ 'This is a hot take').to be_falsy
|
expect(matcher =~ 'This is a hot take').to be_falsy
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'does not match substrings matching keywords' do
|
it 'considers word boundaries when matching' do
|
||||||
KeywordMute.create!(account: alice, keyword: 'take')
|
KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true)
|
||||||
|
|
||||||
expect(matcher =~ 'This is a shiitake mushroom').to be_falsy
|
expect(matcher =~ 'bobcats').to be_falsy
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'matches substrings if whole_word is false' do
|
||||||
|
KeywordMute.create!(account: alice, keyword: 'take', whole_word: false)
|
||||||
|
|
||||||
|
expect(matcher =~ 'This is a shiitake mushroom').to be_truthy
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'matches keywords at the beginning of the text' do
|
it 'matches keywords at the beginning of the text' do
|
||||||
|
|
Loading…
Reference in New Issue