fef/nyastodon
1
1
Fork 0
forked from mirrors/catstodon
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix subscriptions:clear task, refactor feeds, refactor streamable activites

Browse source 
and atom feed generation to some extent, as well as the way mentions are
stored
This commit is contained in:
Eugen Rochko 2016-03-25 02:13:30 +01:00
parent 9594f0e858
commit a08e724476
24 changed files with 219 additions and 234 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/controllers/accounts_controller.rb
View file

@ -11,8 +11,8 @@ class AccountsController < ApplicationController
format.atom do
@entries = @account.stream_entries.order('id desc').with_includes.paginate_by_max_id(20, params[:max_id] || nil)
ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Status' }, activity: [:mentioned_accounts, reblog: :account, thread: :account])
ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Favourite' }, activity: [:account, :thread, :mentioned_accounts])
ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Status' }, activity: [:mentions, reblog: :account, thread: :account])
ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Favourite' }, activity: [:account, :status])
ActiveRecord::Associations::Preloader.new.preload(@entries.select { |a| a.activity_type == 'Follow' }, activity: :target_account)
end
end

6
app/controllers/api/statuses_controller.rb
View file

@ -22,12 +22,10 @@ class Api::StatusesController < ApiController
end
def home
feed = Feed.new(:home, current_user.account)
@statuses = feed.get(20, params[:max_id] || '+inf')
@statuses = Feed.new(:home, current_user.account).get(20, params[:max_id])
end
def mentions
feed = Feed.new(:mentions, current_user.account)
@statuses = feed.get(20, params[:max_id] || '+inf')
@statuses = Feed.new(:mentions, current_user.account).get(20, params[:max_id])
end
end

2
app/controllers/statuses_controller.rb
View file

@ -4,7 +4,7 @@ class StatusesController < ApplicationController
before_action :authenticate_user!
def create
status = PostStatusService.new.(current_user.account, status_params[:text])
PostStatusService.new.(current_user.account, status_params[:text])
redirect_to root_path
rescue ActiveRecord::RecordInvalid
redirect_to root_path

24
app/helpers/application_helper.rb
View file

@ -20,13 +20,25 @@ module ApplicationHelper
end
end
def linkify(status)
mention_hash = {}
status.mentions.each { |m| mention_hash[m.acct] = m }
coder = HTMLEntities.new
def account_from_mentions(search_string, mentions)
mentions.each { |x| return x.account if x.account.acct.eql?(search_string) }
auto_link(coder.encode(status.text), link: :urls, html: { rel: 'nofollow noopener' }).gsub(Account::MENTION_RE) do |m|
account = mention_hash[Account::MENTION_RE.match(m)[1]]
# If that was unsuccessful, try fetching user from db separately
# But this shouldn't ever happen if the mentions were created correctly!
username, domain = search_string.split('@')
if domain == Rails.configuration.x.local_domain
account = Account.find_local(username)
else
account = Account.find_by(username: username, domain: domain)
end
account
end
def linkify(status)
auto_link(HTMLEntities.new.encode(status.text), link: :urls, html: { rel: 'nofollow noopener' }).gsub(Account::MENTION_RE) do |m|
account = account_from_mentions(Account::MENTION_RE.match(m)[1], status.mentions)
"#{m.split('@').first}<a href=\"#{url_for_target(account)}\" class=\"mention\">@<span>#{account.acct}</span></a>"
end.html_safe
end

20
app/helpers/atom_builder_helper.rb
View file

@ -135,6 +135,10 @@ module AtomBuilderHelper
xml.logo url
end
def email(xml, email)
xml.email email
end
def conditionally_formatted(activity)
if activity.is_a?(Status)
content_for_status(activity.reblog? ? activity.reblog : activity)
@ -149,6 +153,7 @@ module AtomBuilderHelper
object_type xml, :person
uri xml, url_for_target(account)
name xml, account.username
email xml, account.local? ? "#{account.acct}@#{Rails.configuration.x.local_domain}" : account.acct
summary xml, account.note
link_alternate xml, url_for_target(account)
link_avatar xml, account
@ -171,16 +176,13 @@ module AtomBuilderHelper
if stream_entry.targeted?
target(xml) do
object_type xml, stream_entry.target.object_type
simple_id xml, uri_for_target(stream_entry.target)
title xml, stream_entry.target.title
link_alternate xml, url_for_target(stream_entry.target)
# People have summary and portable contacts information
if stream_entry.target.object_type == :person
summary xml, stream_entry.target.content
portable_contact xml, stream_entry.target
link_avatar xml, stream_entry.target
include_author xml, stream_entry.target
else
object_type xml, stream_entry.target.object_type
simple_id xml, uri_for_target(stream_entry.target)
title xml, stream_entry.target.title
link_alternate xml, url_for_target(stream_entry.target)
end
# Statuses have content and author

11
app/lib/feed_manager.rb Normal file
View file

@ -0,0 +1,11 @@
class FeedManager
MAX_ITEMS = 800
def self.key(type, id)
"feed:#{type}:#{id}"
end
def self.filter_status?(status, follower)
(status.reply? && !(follower.id = replied_to_user.id || follower.following?(replied_to_user)))
end
end

18
app/models/account.rb
View file

@ -1,4 +1,6 @@
class Account < ActiveRecord::Base
include Targetable
# Local users
has_one :user, inverse_of: :account
validates :username, uniqueness: { scope: :domain, case_sensitive: false }, if: 'local?'
@ -52,18 +54,6 @@ class Account < ActiveRecord::Base
local? ? self.username : "#{self.username}@#{self.domain}"
end
def object_type
:person
end
def title
self.username
end
def content
self.note
end
def subscribed?
!(self.secret.blank? || self.verify_token.blank?)
end
@ -97,6 +87,10 @@ class Account < ActiveRecord::Base
self[:avatar_remote_url] = url
end
def object_type
:person
end
def to_param
self.username
end

31
app/models/concerns/streamable.rb Normal file
View file

@ -0,0 +1,31 @@
module Streamable
extend ActiveSupport::Concern
included do
has_one :stream_entry, as: :activity
def title
super
end
def content
title
end
def target
super
end
def object_type
:activity
end
def thread
super
end
after_create do
self.account.stream_entries.create!(activity: self)
end
end
end

9
app/models/concerns/targetable.rb Normal file
View file

@ -0,0 +1,9 @@
module Targetable
extend ActiveSupport::Concern
included do
def object_type
:object
end
end
end

22
app/models/favourite.rb
View file

@ -1,9 +1,9 @@
class Favourite < ActiveRecord::Base
include Streamable
belongs_to :account, inverse_of: :favourites
belongs_to :status, inverse_of: :favourites
has_one :stream_entry, as: :activity
def verb
:favorite
end
@ -12,27 +12,15 @@ class Favourite < ActiveRecord::Base
"#{self.account.acct} favourited a status by #{self.status.account.acct}"
end
def content
title
end
def object_type
target.object_type
end
def target
def thread
self.status
end
def mentions
[]
end
def thread
target
end
after_create do
self.account.stream_entries.create!(activity: self)
def target
thread
end
end

15
app/models/feed.rb
View file

@ -4,21 +4,24 @@ class Feed
@account = account
end
def get(limit, max_id = '+inf')
def get(limit, max_id)
max_id = '+inf' if max_id.nil?
unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", '-inf', limit: [0, limit])
status_map = Hash.new
# If we're after most recent items and none are there, we need to precompute the feed
return PrecomputeFeedService.new.(@type, @account).take(limit) if unhydrated.empty? && max_id == '+inf'
Status.where(id: unhydrated).with_includes.with_counters.each { |status| status_map[status.id.to_s] = status }
return unhydrated.map { |id| status_map[id] }.compact
if unhydrated.empty? && max_id == '+inf'
PrecomputeFeedService.new.(@type, @account, limit)
else
Status.where(id: unhydrated).with_includes.with_counters.each { |status| status_map[status.id.to_s] = status }
unhydrated.map { |id| status_map[id] }.compact
end
end
private
def key
"feed:#{@type}:#{@account.id}"
FeedManager.key(@type, @account.id)
end
def redis

20
app/models/follow.rb
View file

@ -1,9 +1,9 @@
class Follow < ActiveRecord::Base
include Streamable
belongs_to :account
belongs_to :target_account, class_name: 'Account'
has_one :stream_entry, as: :activity
validates :account, :target_account, presence: true
validates :account_id, uniqueness: { scope: :target_account_id }
@ -16,22 +16,10 @@ class Follow < ActiveRecord::Base
end
def object_type
target.object_type
end
def content
self.destroyed? ? "#{self.account.acct} is no longer following #{self.target_account.acct}" : "#{self.account.acct} started following #{self.target_account.acct}"
:person
end
def title
content
end
def mentions
[]
end
after_create do
self.account.stream_entries.create!(activity: self)
self.destroyed? ? "#{self.account.acct} is no longer following #{self.target_account.acct}" : "#{self.account.acct} started following #{self.target_account.acct}"
end
end

27
app/models/status.rb
View file

@ -1,24 +1,23 @@
class Status < ActiveRecord::Base
include Paginable
include Streamable
belongs_to :account, inverse_of: :statuses
belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies
belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs
has_one :stream_entry, as: :activity
has_many :favourites, inverse_of: :status, dependent: :destroy
has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
has_many :mentioned_accounts, class_name: 'Mention', dependent: :destroy
has_many :mentions, dependent: :destroy
validates :account, presence: true
validates :uri, uniqueness: true, unless: 'local?'
validates :text, presence: true, if: Proc.new { |s| s.local? && !s.reblog? }
scope :with_counters, -> { select('statuses.*, (select count(r.id) from statuses as r where r.reblog_of_id = statuses.id) as reblogs_count, (select count(f.id) from favourites as f where f.status_id = statuses.id) as favourites_count') }
scope :with_includes, -> { includes(:account, :mentioned_accounts, reblog: [:account, :mentioned_accounts], thread: [:account, :mentioned_accounts]) }
scope :with_includes, -> { includes(:account, :mentions, reblog: [:account, :mentions], thread: [:account, :mentions]) }
def local?
self.uri.nil?
@ -60,18 +59,6 @@ class Status < ActiveRecord::Base
self.attributes['favourites_count'] || self.favourites.count
end
def mentions
if @mentions.nil?
@mentions = []
@mentions << thread.account if reply?
@mentions << reblog.account if reblog?
self.mentioned_accounts.each { |mention| @mentions << mention.account } unless reblog?
@mentions = @mentions.uniq
end
@mentions
end
def ancestors
Status.where(id: Status.find_by_sql(['WITH RECURSIVE search_tree(id, in_reply_to_id, path) AS (SELECT id, in_reply_to_id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, statuses.in_reply_to_id, path || statuses.id FROM search_tree JOIN statuses ON statuses.id = search_tree.in_reply_to_id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path DESC', self.id]) - [self])
end
@ -80,7 +67,11 @@ class Status < ActiveRecord::Base
Status.where(id: Status.find_by_sql(['WITH RECURSIVE search_tree(id, path) AS (SELECT id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, path || statuses.id FROM search_tree JOIN statuses ON statuses.in_reply_to_id = search_tree.id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path', self.id]) - [self])
end
after_create do
self.account.stream_entries.create!(activity: self)
def self.as_home_timeline(account)
self.where(account: [account] + account.following).with_includes.with_counters
end
def self.as_mentions_timeline(account)
self.where(id: Mention.where(account: account).pluck(:status_id)).with_includes.with_counters
end
end

2
app/models/stream_entry.rb
View file

@ -41,7 +41,7 @@ class StreamEntry < ActiveRecord::Base
end
def mentions
orphaned? ? [] : self.activity.mentions
self.activity.respond_to?(:mentions) ? self.activity.mentions.map { |x| x.account } : []
end
private

18
app/services/fan_out_on_write_service.rb
View file

@ -1,6 +1,4 @@
class FanOutOnWriteService < BaseService
MAX_FEED_SIZE = 800
# Push a status into home and mentions feeds
# @param [Status] status
def call(status)
@ -17,13 +15,13 @@ class FanOutOnWriteService < BaseService
def deliver_to_followers(status, replied_to_user)
status.account.followers.each do |follower|
next if (status.reply? && !(follower.id = replied_to_user.id || follower.following?(replied_to_user))) || !follower.local?
next if !follower.local? || FeedManager.filter_status?(status, follower)
push(:home, follower.id, status)
end
end
def deliver_to_mentioned(status)
status.mentioned_accounts.each do |mention|
status.mentions.each do |mention|
mentioned_account = mention.account
next unless mentioned_account.local?
push(:mentions, mentioned_account.id, status)
@ -31,19 +29,15 @@ class FanOutOnWriteService < BaseService
end
def push(type, receiver_id, status)
redis.zadd(key(type, receiver_id), status.id, status.id)
redis.zadd(FeedManager.key(type, receiver_id), status.id, status.id)
trim(type, receiver_id)
end
def trim(type, receiver_id)
return unless redis.zcard(key(type, receiver_id)) > MAX_FEED_SIZE
return unless redis.zcard(FeedManager.key(type, receiver_id)) > FeedManager::MAX_ITEMS
last = redis.zrevrange(key(type, receiver_id), MAX_FEED_SIZE - 1, MAX_FEED_SIZE - 1)
redis.zremrangebyscore(key(type, receiver_id), '-inf', "(#{last.last}")
end
def key(type, id)
"feed:#{type}:#{id}"
last = redis.zrevrange(FeedManager.key(type, receiver_id), FeedManager::MAX_ITEMS - 1, FeedManager::MAX_ITEMS - 1)
redis.zremrangebyscore(FeedManager.key(type, receiver_id), '-inf', "(#{last.last}")
end
def redis

34
app/services/precompute_feed_service.rb
View file

@ -1,34 +1,22 @@
class PrecomputeFeedService < BaseService
MAX_FEED_SIZE = 800
# Fill up a user's home/mentions feed from DB and return it
# Fill up a user's home/mentions feed from DB and return a subset
# @param [Symbol] type :home or :mentions
# @param [Account] account
# @return [Array]
def call(type, account)
statuses = send(type.to_s, account).order('created_at desc').limit(MAX_FEED_SIZE)
statuses.each { |status| push(type, account.id, status) }
statuses
def call(type, account, limit)
instant_return = []
Status.send("as_#{type}_timeline", account).order('created_at desc').limit(FeedManager::MAX_ITEMS).each do |status|
next if type == :home && FeedManager.filter_status?(status, account)
redis.zadd(FeedManager.key(type, receiver_id), status.id, status.id)
instant_return << status unless instant_return.size > limit
end
instant_return
end
private
def push(type, receiver_id, status)
redis.zadd(key(type, receiver_id), status.id, status.id)
end
def home(account)
Status.where(account: [account] + account.following).with_includes.with_counters
end
def mentions(account)
Status.where(id: Mention.where(account: account).pluck(:status_id)).with_includes.with_counters
end
def key(type, id)
"feed:#{type}:#{id}"
end
def redis
$redis
end

112
app/services/process_feed_service.rb
View file

@ -4,65 +4,67 @@ class ProcessFeedService < BaseService
# @param [Account] account Account this feed belongs to
def call(body, account)
xml = Nokogiri::XML(body)
# If we got a full feed, make sure the account's profile is up to date
unless xml.at_xpath('/xmlns:feed').nil?
update_remote_profile_service.(xml.at_xpath('/xmlns:feed/xmlns:author'), account)
end
# Process entries
xml.xpath('//xmlns:entry').each do |entry|
next unless [:note, :comment, :activity].include? object_type(entry)
status = Status.find_by(uri: activity_id(entry))
# If we already have a post and the verb is now "delete", we gotta delete it and move on!
if !status.nil? && verb(entry) == :delete
delete_post!(status)
next
end
next unless status.nil?
status = Status.new(uri: activity_id(entry), url: activity_link(entry), account: account, text: content(entry), created_at: published(entry), updated_at: updated(entry))
if verb(entry) == :share
add_reblog!(entry, status)
elsif verb(entry) == :post
if thread_id(entry).nil?
add_post!(entry, status)
else
add_reply!(entry, status)
end
end
# If we added a status, go through accounts it mentions and create respective relations
unless status.new_record?
entry.xpath('./xmlns:link[@rel="mentioned"]').each do |mention_link|
# Here we have to do a reverse lookup of local accounts by their URL!
# It's not pretty at all! I really wish all these protocols sticked to
# using acct:username@domain only! It would make things so much easier
# and tidier
href = Addressable::URI.parse(mention_link.attribute('href').value)
if href.host == Rails.configuration.x.local_domain
mentioned_account = Account.find_local(href.path.gsub('/users/', ''))
unless mentioned_account.nil?
mentioned_account.mentions.where(status: status).first_or_create(status: status)
NotificationMailer.mention(mentioned_account, status).deliver_later
end
end
end
fan_out_on_write_service.(status)
end
end
update_remote_profile_service.(xml.at_xpath('/xmlns:feed/xmlns:author'), account) unless xml.at_xpath('/xmlns:feed').nil?
xml.xpath('//xmlns:entry').each { |entry| process_entry(account, entry) }
end
private
def process_entry(account, entry)
return unless [:note, :comment, :activity].include? object_type(entry)
status = Status.find_by(uri: activity_id(entry))
# If we already have a post and the verb is now "delete", we gotta delete it and move on!
if !status.nil? && verb(entry) == :delete
delete_post!(status)
return
end
return unless status.nil?
status = Status.new(uri: activity_id(entry), url: activity_link(entry), account: account, text: content(entry), created_at: published(entry), updated_at: updated(entry))
if verb(entry) == :share
add_reblog!(entry, status)
elsif verb(entry) == :post
if thread_id(entry).nil?
add_post!(entry, status)
else
add_reply!(entry, status)
end
end
# If we added a status, go through accounts it mentions and create respective relations
unless status.new_record?
record_remote_mentions(status, entry.xpath('./xmlns:link[@rel="mentioned"]'))
fan_out_on_write_service.(status)
end
end
def record_remote_mentions(status, links)
# Here we have to do a reverse lookup of local accounts by their URL!
# It's not pretty at all! I really wish all these protocols sticked to
# using acct:username@domain only! It would make things so much easier
# and tidier
links.each do |mention_link|
href = Addressable::URI.parse(mention_link.attribute('href').value)
if href.host == Rails.configuration.x.local_domain
# A local user is mentioned
mentioned_account = Account.find_local(href.path.gsub('/users/', ''))
unless mentioned_account.nil?
mentioned_account.mentions.where(status: status).first_or_create(status: status)
NotificationMailer.mention(mentioned_account, status).deliver_later
end
else
# What to do about remote user?
end
end
end
def add_post!(_entry, status)
status.save!
end

2
app/services/process_mentions_service.rb
View file

@ -18,7 +18,7 @@ class ProcessMentionsService < BaseService
mentioned_account.mentions.where(status: status).first_or_create(status: status)
end
status.mentioned_accounts.each do |mention|
status.mentions.each do |mention|
mentioned_account = mention.account
if mentioned_account.local?

2
lib/tasks/subscriptions.rake
View file

@ -5,7 +5,7 @@ namespace :subscriptions do
accounts = Account.where('(select count(f.id) from follows as f where f.target_account_id = accounts.id) = 0').where.not(domain: nil)
accounts.each do |a|
a.subscription(api_subscription_url(a.id)).unsubscribe
a.subscription('').unsubscribe
a.update!(verify_token: '', secret: '')
end
end

25
spec/helpers/application_helper_spec.rb
View file

@ -30,6 +30,29 @@ RSpec.describe ApplicationHelper, type: :helper do
end
describe '#linkify' do
pending
let(:alice) { Fabricate(:account, username: 'alice') }
let(:bob) { Fabricate(:account, username: 'bob', domain: 'example.com', url: 'http://example.com/bob') }
it 'turns mention of remote user into link' do
status = Fabricate(:status, text: 'Hello @bob@example.com', account: bob)
status.mentions.create(account: bob)
expect(helper.linkify(status)).to match('<a href="http://example.com/bob" class="mention">@<span>bob@example.com</span></a>')
end
it 'turns mention of local user into link' do
status = Fabricate(:status, text: 'Hello @alice', account: bob)
status.mentions.create(account: alice)
expect(helper.linkify(status)).to match('<a href="http://test.host/users/alice" class="mention">@<span>alice</span></a>')
end
end
describe '#account_from_mentions' do
let(:bob) { Fabricate(:account, username: 'bob', domain: 'example.com') }
let(:status) { Fabricate(:status, text: 'Hello @bob@example.com', account: bob) }
let(:mentions) { [Mention.create(status: status, account: bob)] }
it 'returns account' do
expect(helper.account_from_mentions('bob@example.com', mentions)).to eq bob
end
end
end

12
spec/models/account_spec.rb
View file

@ -96,18 +96,6 @@ RSpec.describe Account, type: :model do
end
end
describe '#title' do
it 'is the same as the username' do
expect(subject.title).to eql subject.username
end
end
describe '#content' do
it 'is the same as the note' do
expect(subject.content).to eql subject.note
end
end
describe '#ping!' do
pending
end

6
spec/models/favourite_spec.rb
View file

@ -42,12 +42,6 @@ RSpec.describe Favourite, type: :model do
end
end
describe '#mentions' do
it 'is always empty' do
expect(subject.mentions).to be_empty
end
end
describe '#thread' do
it 'equals the target' do
expect(subject.thread).to eq subject.target

6
spec/models/follow_spec.rb
View file

@ -35,10 +35,4 @@ RSpec.describe Follow, type: :model do
expect(subject.target).to eq bob
end
end
describe '#mentions' do
it 'is empty' do
expect(subject.mentions).to be_empty
end
end
end

25
spec/models/status_spec.rb
View file

@ -40,31 +40,6 @@ RSpec.describe Status, type: :model do
end
end
describe '#mentions' do
before do
bob # make sure the account exists
end
it 'is empty if the status is self-contained and does not mention anyone' do
expect(subject.mentions).to be_empty
end
it 'returns mentioned accounts' do
subject.mentioned_accounts.create!(account: bob)
expect(subject.mentions).to include bob
end
it 'returns account of the replied-to status' do
subject.thread = other
expect(subject.mentions).to include bob
end
it 'returns the account of the shared status' do
subject.reblog = other
expect(subject.mentions).to include bob
end
end
describe '#verb' do
it 'is always post' do
expect(subject.verb).to be :post