Fix subscriptions:clear task, refactor feeds, refactor streamable activites

and atom feed generation to some extent, as well as the way mentions are
stored

app/controllers/accounts_controller.rb

@@ -11,8 +11,8 @@ class AccountsController < ApplicationController
       format.atom do
         @entries = @account.stream_entries.order('id desc').with_includes.paginate_by_max_id(20, params[:max_id] || nil)
 
-        ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Status' }, activity: [:mentioned_accounts, reblog: :account, thread: :account])
-        ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Favourite' }, activity: [:account, :thread, :mentioned_accounts])
+        ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Status' }, activity: [:mentions, reblog: :account, thread: :account])
+        ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Favourite' }, activity: [:account, :status])
         ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Follow' }, activity: :target_account)
       end
     end

app/controllers/api/statuses_controller.rb

@@ -22,12 +22,10 @@ class Api::StatusesController < ApiController
   end
 
   def home
-    feed      = Feed.new(:home, current_user.account)
-    @statuses = feed.get(20, params[:max_id] || '+inf')
+    @statuses = Feed.new(:home, current_user.account).get(20, params[:max_id])
   end
 
   def mentions
-    feed      = Feed.new(:mentions, current_user.account)
-    @statuses = feed.get(20, params[:max_id] || '+inf')
+    @statuses = Feed.new(:mentions, current_user.account).get(20, params[:max_id])
   end
 end

app/controllers/statuses_controller.rb

@@ -4,7 +4,7 @@ class StatusesController < ApplicationController
   before_action :authenticate_user!
 
   def create
-    status = PostStatusService.new.(current_user.account, status_params[:text])
+    PostStatusService.new.(current_user.account, status_params[:text])
     redirect_to root_path
   rescue ActiveRecord::RecordInvalid
     redirect_to root_path

app/helpers/application_helper.rb

@@ -20,13 +20,25 @@ module ApplicationHelper
     end
   end
 
-  def linkify(status)
-    mention_hash = {}
-    status.mentions.each { |m| mention_hash[m.acct] = m }
-    coder = HTMLEntities.new
+  def account_from_mentions(search_string, mentions)
+    mentions.each { |x| return x.account if x.account.acct.eql?(search_string) }
 
-    auto_link(coder.encode(status.text), link: :urls, html: { rel: 'nofollow noopener' }).gsub(Account::MENTION_RE) do |m|
-      account = mention_hash[Account::MENTION_RE.match(m)[1]]
+    # If that was unsuccessful, try fetching user from db separately
+    # But this shouldn't ever happen if the mentions were created correctly!
+    username, domain = search_string.split('@')
+
+    if domain == Rails.configuration.x.local_domain
+      account = Account.find_local(username)
+    else
+      account = Account.find_by(username: username, domain: domain)
+    end
+
+    account
+  end
+
+  def linkify(status)
+    auto_link(HTMLEntities.new.encode(status.text), link: :urls, html: { rel: 'nofollow noopener' }).gsub(Account::MENTION_RE) do |m|
+      account = account_from_mentions(Account::MENTION_RE.match(m)[1], status.mentions)
       "#{m.split('@').first}<a href=\"#{url_for_target(account)}\" class=\"mention\">@<span>#{account.acct}</span></a>"
     end.html_safe
   end

app/helpers/atom_builder_helper.rb

@@ -135,6 +135,10 @@ module AtomBuilderHelper
     xml.logo url
   end
 
+  def email(xml, email)
+    xml.email email
+  end
+
   def conditionally_formatted(activity)
     if activity.is_a?(Status)
       content_for_status(activity.reblog? ? activity.reblog : activity)
@@ -149,6 +153,7 @@ module AtomBuilderHelper
     object_type      xml, :person
     uri              xml, url_for_target(account)
     name             xml, account.username
+    email            xml, account.local? ? "#{account.acct}@#{Rails.configuration.x.local_domain}" : account.acct
     summary          xml, account.note
     link_alternate   xml, url_for_target(account)
     link_avatar      xml, account
@@ -171,16 +176,13 @@ module AtomBuilderHelper
 
     if stream_entry.targeted?
       target(xml) do
+        if stream_entry.target.object_type == :person
+          include_author xml, stream_entry.target
+        else
           object_type    xml, stream_entry.target.object_type
           simple_id      xml, uri_for_target(stream_entry.target)
           title          xml, stream_entry.target.title
           link_alternate xml, url_for_target(stream_entry.target)
-
-        # People have summary and portable contacts information
-        if stream_entry.target.object_type == :person
-          summary          xml, stream_entry.target.content
-          portable_contact xml, stream_entry.target
-          link_avatar      xml, stream_entry.target
         end
 
         # Statuses have content and author

app/lib/feed_manager.rb

@@ -0,0 +1,11 @@
+class FeedManager
+  MAX_ITEMS = 800
+
+  def self.key(type, id)
+    "feed:#{type}:#{id}"
+  end
+
+  def self.filter_status?(status, follower)
+    (status.reply? && !(follower.id = replied_to_user.id || follower.following?(replied_to_user)))
+  end
+end

app/models/account.rb

@@ -1,4 +1,6 @@
 class Account < ActiveRecord::Base
+  include Targetable
+
   # Local users
   has_one :user, inverse_of: :account
   validates :username, uniqueness: { scope: :domain, case_sensitive: false }, if:     'local?'
@@ -52,18 +54,6 @@ class Account < ActiveRecord::Base
     local? ? self.username : "#{self.username}@#{self.domain}"
   end
 
-  def object_type
-    :person
-  end
-
-  def title
-    self.username
-  end
-
-  def content
-    self.note
-  end
-
   def subscribed?
     !(self.secret.blank? || self.verify_token.blank?)
   end
@@ -97,6 +87,10 @@ class Account < ActiveRecord::Base
     self[:avatar_remote_url] = url
   end
 
+  def object_type
+    :person
+  end
+
   def to_param
     self.username
   end

app/models/concerns/streamable.rb

@@ -0,0 +1,31 @@
+module Streamable
+  extend ActiveSupport::Concern
+
+  included do
+    has_one :stream_entry, as: :activity
+
+    def title
+      super
+    end
+
+    def content
+      title
+    end
+
+    def target
+      super
+    end
+
+    def object_type
+      :activity
+    end
+
+    def thread
+      super
+    end
+
+    after_create do
+      self.account.stream_entries.create!(activity: self)
+    end
+  end
+end

app/models/concerns/targetable.rb

@@ -0,0 +1,9 @@
+module Targetable
+  extend ActiveSupport::Concern
+
+  included do
+    def object_type
+      :object
+    end
+  end
+end

app/models/favourite.rb

@@ -1,9 +1,9 @@
 class Favourite < ActiveRecord::Base
+  include Streamable
+
   belongs_to :account, inverse_of: :favourites
   belongs_to :status,  inverse_of: :favourites
 
-  has_one :stream_entry, as: :activity
-
   def verb
     :favorite
   end
@@ -12,27 +12,15 @@ class Favourite < ActiveRecord::Base
     "#{self.account.acct} favourited a status by #{self.status.account.acct}"
   end
 
-  def content
-    title
-  end
-
   def object_type
     target.object_type
   end
 
-  def target
+  def thread
     self.status
   end
 
-  def mentions
-    []
-  end
-
-  def thread
-    target
-  end
-
-  after_create do
-    self.account.stream_entries.create!(activity: self)
+  def target
+    thread
   end
 end

app/models/feed.rb

@@ -4,21 +4,24 @@ class Feed
     @account = account
   end
 
-  def get(limit, max_id = '+inf')
+  def get(limit, max_id)
+    max_id     = '+inf' if max_id.nil?
     unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", '-inf', limit: [0, limit])
     status_map = Hash.new
 
     # If we're after most recent items and none are there, we need to precompute the feed
-    return PrecomputeFeedService.new.(@type, @account).take(limit) if unhydrated.empty? && max_id == '+inf'
-
+    if unhydrated.empty? && max_id == '+inf'
+      PrecomputeFeedService.new.(@type, @account, limit)
+    else
       Status.where(id: unhydrated).with_includes.with_counters.each { |status| status_map[status.id.to_s] = status }
-    return unhydrated.map { |id| status_map[id] }.compact
+      unhydrated.map { |id| status_map[id] }.compact
+    end
   end
 
   private
 
   def key
-    "feed:#{@type}:#{@account.id}"
+    FeedManager.key(@type, @account.id)
   end
 
   def redis

app/models/follow.rb

@@ -1,9 +1,9 @@
 class Follow < ActiveRecord::Base
+  include Streamable
+
   belongs_to :account
   belongs_to :target_account, class_name: 'Account'
 
-  has_one :stream_entry, as: :activity
-
   validates :account, :target_account, presence: true
   validates :account_id, uniqueness: { scope: :target_account_id }
 
@@ -16,22 +16,10 @@ class Follow < ActiveRecord::Base
   end
 
   def object_type
-    target.object_type
-  end
-
-  def content
-    self.destroyed? ? "#{self.account.acct} is no longer following #{self.target_account.acct}" : "#{self.account.acct} started following #{self.target_account.acct}"
+    :person
   end
 
   def title
-    content
-  end
-
-  def mentions
-    []
-  end
-
-  after_create do
-    self.account.stream_entries.create!(activity: self)
+    self.destroyed? ? "#{self.account.acct} is no longer following #{self.target_account.acct}" : "#{self.account.acct} started following #{self.target_account.acct}"
   end
 end

app/models/status.rb

@@ -1,24 +1,23 @@
 class Status < ActiveRecord::Base
   include Paginable
+  include Streamable
 
   belongs_to :account, inverse_of: :statuses
 
   belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs
 
-  has_one :stream_entry, as: :activity
-
   has_many :favourites, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
-  has_many :mentioned_accounts, class_name: 'Mention', dependent: :destroy
+  has_many :mentions, dependent: :destroy
 
   validates :account, presence: true
   validates :uri, uniqueness: true, unless: 'local?'
   validates :text, presence: true, if: Proc.new { |s| s.local? && !s.reblog? }
 
   scope :with_counters, -> { select('statuses.*, (select count(r.id) from statuses as r where r.reblog_of_id = statuses.id) as reblogs_count, (select count(f.id) from favourites as f where f.status_id = statuses.id) as favourites_count') }
-  scope :with_includes, -> { includes(:account, :mentioned_accounts, reblog: [:account, :mentioned_accounts], thread: [:account, :mentioned_accounts]) }
+  scope :with_includes, -> { includes(:account, :mentions, reblog: [:account, :mentions], thread: [:account, :mentions]) }
 
   def local?
     self.uri.nil?
@@ -60,18 +59,6 @@ class Status < ActiveRecord::Base
     self.attributes['favourites_count'] || self.favourites.count
   end
 
-  def mentions
-    if @mentions.nil?
-      @mentions = []
-      @mentions << thread.account if reply?
-      @mentions << reblog.account if reblog?
-      self.mentioned_accounts.each { |mention| @mentions << mention.account } unless reblog?
-      @mentions = @mentions.uniq
-    end
-
-    @mentions
-  end
-
   def ancestors
     Status.where(id: Status.find_by_sql(['WITH RECURSIVE search_tree(id, in_reply_to_id, path) AS (SELECT id, in_reply_to_id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, statuses.in_reply_to_id, path || statuses.id FROM search_tree JOIN statuses ON statuses.id = search_tree.in_reply_to_id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path DESC', self.id]) - [self])
   end
@@ -80,7 +67,11 @@ class Status < ActiveRecord::Base
     Status.where(id: Status.find_by_sql(['WITH RECURSIVE search_tree(id, path) AS (SELECT id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, path || statuses.id FROM search_tree JOIN statuses ON statuses.in_reply_to_id = search_tree.id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path', self.id]) - [self])
   end
 
-  after_create do
-    self.account.stream_entries.create!(activity: self)
+  def self.as_home_timeline(account)
+    self.where(account: [account] + account.following).with_includes.with_counters
+  end
+
+  def self.as_mentions_timeline(account)
+    self.where(id: Mention.where(account: account).pluck(:status_id)).with_includes.with_counters
   end
 end

app/models/stream_entry.rb

@@ -41,7 +41,7 @@ class StreamEntry < ActiveRecord::Base
   end
 
   def mentions
-    orphaned? ? [] : self.activity.mentions
+    self.activity.respond_to?(:mentions) ? self.activity.mentions.map { |x| x.account } : []
   end
 
   private

app/services/fan_out_on_write_service.rb

@@ -1,6 +1,4 @@
 class FanOutOnWriteService < BaseService
-  MAX_FEED_SIZE = 800
-
   # Push a status into home and mentions feeds
   # @param [Status] status
   def call(status)
@@ -17,13 +15,13 @@ class FanOutOnWriteService < BaseService
 
   def deliver_to_followers(status, replied_to_user)
     status.account.followers.each do |follower|
-      next if (status.reply? && !(follower.id = replied_to_user.id || follower.following?(replied_to_user))) || !follower.local?
+      next if !follower.local? || FeedManager.filter_status?(status, follower)
       push(:home, follower.id, status)
     end
   end
 
   def deliver_to_mentioned(status)
-    status.mentioned_accounts.each do |mention|
+    status.mentions.each do |mention|
       mentioned_account = mention.account
       next unless mentioned_account.local?
       push(:mentions, mentioned_account.id, status)
@@ -31,19 +29,15 @@ class FanOutOnWriteService < BaseService
   end
 
   def push(type, receiver_id, status)
-    redis.zadd(key(type, receiver_id), status.id, status.id)
+    redis.zadd(FeedManager.key(type, receiver_id), status.id, status.id)
     trim(type, receiver_id)
   end
 
   def trim(type, receiver_id)
-    return unless redis.zcard(key(type, receiver_id)) > MAX_FEED_SIZE
+    return unless redis.zcard(FeedManager.key(type, receiver_id)) > FeedManager::MAX_ITEMS
 
-    last = redis.zrevrange(key(type, receiver_id), MAX_FEED_SIZE - 1, MAX_FEED_SIZE - 1)
-    redis.zremrangebyscore(key(type, receiver_id), '-inf', "(#{last.last}")
-  end
-
-  def key(type, id)
-    "feed:#{type}:#{id}"
+    last = redis.zrevrange(FeedManager.key(type, receiver_id), FeedManager::MAX_ITEMS - 1, FeedManager::MAX_ITEMS - 1)
+    redis.zremrangebyscore(FeedManager.key(type, receiver_id), '-inf', "(#{last.last}")
   end
 
   def redis

app/services/precompute_feed_service.rb

@@ -1,34 +1,22 @@
 class PrecomputeFeedService < BaseService
-  MAX_FEED_SIZE = 800
-
-  # Fill up a user's home/mentions feed from DB and return it
+  # Fill up a user's home/mentions feed from DB and return a subset
   # @param [Symbol] type :home or :mentions
   # @param [Account] account
   # @return [Array]
-  def call(type, account)
-    statuses = send(type.to_s, account).order('created_at desc').limit(MAX_FEED_SIZE)
-    statuses.each { |status| push(type, account.id, status) }
-    statuses
+  def call(type, account, limit)
+    instant_return = []
+
+    Status.send("as_#{type}_timeline", account).order('created_at desc').limit(FeedManager::MAX_ITEMS).each do |status|
+      next if type == :home && FeedManager.filter_status?(status, account)
+      redis.zadd(FeedManager.key(type, receiver_id), status.id, status.id)
+      instant_return << status unless instant_return.size > limit
+    end
+
+    instant_return
   end
 
   private
 
-  def push(type, receiver_id, status)
-    redis.zadd(key(type, receiver_id), status.id, status.id)
-  end
-
-  def home(account)
-    Status.where(account: [account] + account.following).with_includes.with_counters
-  end
-
-  def mentions(account)
-    Status.where(id: Mention.where(account: account).pluck(:status_id)).with_includes.with_counters
-  end
-
-  def key(type, id)
-    "feed:#{type}:#{id}"
-  end
-
   def redis
     $redis
   end

app/services/process_feed_service.rb

@@ -4,25 +4,24 @@ class ProcessFeedService < BaseService
   # @param [Account] account Account this feed belongs to
   def call(body, account)
     xml = Nokogiri::XML(body)
-
-    # If we got a full feed, make sure the account's profile is up to date
-    unless xml.at_xpath('/xmlns:feed').nil?
-      update_remote_profile_service.(xml.at_xpath('/xmlns:feed/xmlns:author'), account)
+    update_remote_profile_service.(xml.at_xpath('/xmlns:feed/xmlns:author'), account) unless xml.at_xpath('/xmlns:feed').nil?
+    xml.xpath('//xmlns:entry').each { |entry| process_entry(account, entry) }
   end
 
-    # Process entries
-    xml.xpath('//xmlns:entry').each do |entry|
-      next unless [:note, :comment, :activity].include? object_type(entry)
+  private
+
+  def process_entry(account, entry)
+    return unless [:note, :comment, :activity].include? object_type(entry)
 
     status = Status.find_by(uri: activity_id(entry))
 
     # If we already have a post and the verb is now "delete", we gotta delete it and move on!
     if !status.nil? && verb(entry) == :delete
       delete_post!(status)
-        next
+      return
     end
 
-      next unless status.nil?
+    return unless status.nil?
 
     status = Status.new(uri: activity_id(entry), url: activity_link(entry), account: account, text: content(entry), created_at: published(entry), updated_at: updated(entry))
 
@@ -38,31 +37,34 @@ class ProcessFeedService < BaseService
 
     # If we added a status, go through accounts it mentions and create respective relations
     unless status.new_record?
-        entry.xpath('./xmlns:link[@rel="mentioned"]').each do |mention_link|
+      record_remote_mentions(status, entry.xpath('./xmlns:link[@rel="mentioned"]'))
+      fan_out_on_write_service.(status)
+    end
+  end
+
+  def record_remote_mentions(status, links)
     # Here we have to do a reverse lookup of local accounts by their URL!
     # It's not pretty at all! I really wish all these protocols sticked to
     # using acct:username@domain only! It would make things so much easier
     # and tidier
 
+    links.each do |mention_link|
       href = Addressable::URI.parse(mention_link.attribute('href').value)
 
       if href.host == Rails.configuration.x.local_domain
+        # A local user is mentioned
         mentioned_account = Account.find_local(href.path.gsub('/users/', ''))
 
         unless mentioned_account.nil?
           mentioned_account.mentions.where(status: status).first_or_create(status: status)
           NotificationMailer.mention(mentioned_account, status).deliver_later
         end
-          end
-        end
-
-        fan_out_on_write_service.(status)
+      else
+        # What to do about remote user?
       end
     end
   end
 
-  private
-
   def add_post!(_entry, status)
     status.save!
   end

app/services/process_mentions_service.rb

@@ -18,7 +18,7 @@ class ProcessMentionsService < BaseService
       mentioned_account.mentions.where(status: status).first_or_create(status: status)
     end
 
-    status.mentioned_accounts.each do |mention|
+    status.mentions.each do |mention|
       mentioned_account = mention.account
 
       if mentioned_account.local?

lib/tasks/subscriptions.rake

@@ -5,7 +5,7 @@ namespace :subscriptions do
     accounts = Account.where('(select count(f.id) from follows as f where f.target_account_id = accounts.id) = 0').where.not(domain: nil)
 
     accounts.each do |a|
-      a.subscription(api_subscription_url(a.id)).unsubscribe
+      a.subscription('').unsubscribe
       a.update!(verify_token: '', secret: '')
     end
   end

spec/helpers/application_helper_spec.rb

@@ -30,6 +30,29 @@ RSpec.describe ApplicationHelper, type: :helper do
   end
 
   describe '#linkify' do
-    pending
+    let(:alice) { Fabricate(:account, username: 'alice') }
+    let(:bob) { Fabricate(:account, username: 'bob', domain: 'example.com', url: 'http://example.com/bob') }
+
+    it 'turns mention of remote user into link' do
+      status = Fabricate(:status, text: 'Hello @bob@example.com', account: bob)
+      status.mentions.create(account: bob)
+      expect(helper.linkify(status)).to match('<a href="http://example.com/bob" class="mention">@<span>bob@example.com</span></a>')
+    end
+
+    it 'turns mention of local user into link' do
+      status = Fabricate(:status, text: 'Hello @alice', account: bob)
+      status.mentions.create(account: alice)
+      expect(helper.linkify(status)).to match('<a href="http://test.host/users/alice" class="mention">@<span>alice</span></a>')
+    end
+  end
+
+  describe '#account_from_mentions' do
+    let(:bob) { Fabricate(:account, username: 'bob', domain: 'example.com') }
+    let(:status) { Fabricate(:status, text: 'Hello @bob@example.com', account: bob) }
+    let(:mentions) { [Mention.create(status: status, account: bob)] }
+
+    it 'returns account' do
+      expect(helper.account_from_mentions('bob@example.com', mentions)).to eq bob
+    end
   end
 end

spec/models/account_spec.rb

@@ -96,18 +96,6 @@ RSpec.describe Account, type: :model do
     end
   end
 
-  describe '#title' do
-    it 'is the same as the username' do
-      expect(subject.title).to eql subject.username
-    end
-  end
-
-  describe '#content' do
-    it 'is the same as the note' do
-      expect(subject.content).to eql subject.note
-    end
-  end
-
   describe '#ping!' do
     pending
   end

spec/models/favourite_spec.rb

@@ -42,12 +42,6 @@ RSpec.describe Favourite, type: :model do
     end
   end
 
-  describe '#mentions' do
-    it 'is always empty' do
-      expect(subject.mentions).to be_empty
-    end
-  end
-
   describe '#thread' do
     it 'equals the target' do
       expect(subject.thread).to eq subject.target

spec/models/follow_spec.rb

@@ -35,10 +35,4 @@ RSpec.describe Follow, type: :model do
       expect(subject.target).to eq bob
     end
   end
-
-  describe '#mentions' do
-    it 'is empty' do
-      expect(subject.mentions).to be_empty
-    end
-  end
 end

spec/models/status_spec.rb

@@ -40,31 +40,6 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe '#mentions' do
-    before do
-      bob # make sure the account exists
-    end
-
-    it 'is empty if the status is self-contained and does not mention anyone' do
-      expect(subject.mentions).to be_empty
-    end
-
-    it 'returns mentioned accounts' do
-      subject.mentioned_accounts.create!(account: bob)
-      expect(subject.mentions).to include bob
-    end
-
-    it 'returns account of the replied-to status' do
-      subject.thread = other
-      expect(subject.mentions).to include bob
-    end
-
-    it 'returns the account of the shared status' do
-      subject.reblog = other
-      expect(subject.mentions).to include bob
-    end
-  end
-
   describe '#verb' do
     it 'is always post' do
       expect(subject.verb).to be :post