Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Memri
plugins
mastodon
Commits
5ec6ee05
Commit
5ec6ee05
authored
4 years ago
by
angella
Browse files
Options
Download
Email Patches
Plain Diff
Cleaned up function nsming
parent
120d1012
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
twitter_importer/importer/pod_controller.py
+79
-114
twitter_importer/importer/pod_controller.py
twitter_importer/twitter_plugin.py
+27
-6
twitter_importer/twitter_plugin.py
with
106 additions
and
120 deletions
+106
-120
twitter_importer/importer/pod_controller.py
+
79
-
114
View file @
5ec6ee05
...
...
@@ -279,16 +279,16 @@ class PodController:
all_accounts
=
self
.
client
.
search
({
"type"
:
"Account"
})
# check that account doesn't exist before adding new one
if
any
(
item
.
externalId
==
user_dict
[
"id_str"
]
for
item
in
all_accounts
):
updated_account
=
self
.
handle_existing
_user
(
updated_account
=
self
.
update
_user
(
user_dict
,
account_to_add
,
person_to_add
,
all_accounts
)
return
updated_account
else
:
new_account
=
self
.
handle_new
_user
(
user_dict
,
account_to_add
,
person_to_add
)
new_account
=
self
.
add
_user
(
user_dict
,
account_to_add
,
person_to_add
)
return
new_account
def
handle_new
_user
(
self
,
user_dict
,
account_to_add
,
person_to_add
):
def
add
_user
(
self
,
user_dict
,
account_to_add
,
person_to_add
):
"""
Takes in the user dictionary, user account_to_add object to add, and person_to_add object.
It adds it to the pod and creates an edge between account and person.
...
...
@@ -328,9 +328,7 @@ class PodController:
person_from_db
=
self
.
add_person
(
person_to_add
,
account_to_add
)
return
account_to_add
def
handle_existing_user
(
self
,
user_dict
,
account_to_add
,
person_to_add
,
all_accounts
):
def
update_user
(
self
,
user_dict
,
account_to_add
,
person_to_add
,
all_accounts
):
"""
Takes in the user dictionary, user account_to_add object to add, and person_to_add object.
It updates countries and person for the given user account.
...
...
@@ -362,7 +360,7 @@ class PodController:
)
return
updated_account
def
handle
_color
(
self
,
color
,
associated_item
,
edge_name
):
def
add
_color
(
self
,
color
,
associated_item
,
edge_name
):
"""
Takes in the color of a given property, associated_item and edge_name,
adds it to the pod and creates an edge between associated_item and color.
...
...
@@ -396,14 +394,14 @@ class PodController:
edge_success
=
self
.
client
.
create_edge
(
edge
)
return
color_to_add
def
add_account_styles
(
self
,
user_dict
,
a
ssociated_item
):
def
add_account_styles
(
self
,
user_dict
,
a
ccount
):
"""
Takes in user_dict dictionary, a
ssociated_item
,
Checks for exisiting colors in user_dict and gives them to
handle
_color.
Takes in user_dict dictionary, a
ccount
,
Checks for exisiting colors in user_dict and gives them to
add
_color.
Parameters:
user_dict (dict): User dictionary.
a
ssociated_item
(object): Item using the Color
a
ccount
(object): Item using the Color
Returns:
"""
# Add profile_background_color
...
...
@@ -411,9 +409,9 @@ class PodController:
"profile_background_color"
in
user_dict
and
user_dict
[
"profile_background_color"
]
!=
"None"
):
profile_background_color
=
self
.
handle
_color
(
profile_background_color
=
self
.
add
_color
(
user_dict
[
"profile_background_color"
],
a
ssociated_item
,
a
ccount
,
"profileBackgroundColor"
,
)
# Add profile_link_color
...
...
@@ -421,17 +419,17 @@ class PodController:
"profile_link_color"
in
user_dict
and
user_dict
[
"profile_link_color"
]
!=
"None"
):
profile_link_color
=
self
.
handle
_color
(
user_dict
[
"profile_link_color"
],
a
ssociated_item
,
"profileLinkColor"
,
profile_link_color
=
self
.
add
_color
(
user_dict
[
"profile_link_color"
],
a
ccount
,
"profileLinkColor"
,
)
# Add profile_sidebar_border_color
if
(
"profile_sidebar_border_color"
in
user_dict
and
user_dict
[
"profile_sidebar_border_color"
]
!=
"None"
):
profile_sidebar_border_color
=
self
.
handle
_color
(
profile_sidebar_border_color
=
self
.
add
_color
(
user_dict
[
"profile_sidebar_border_color"
],
a
ssociated_item
,
a
ccount
,
"profileSidebarBorderColor"
,
)
# Add profile_sidebar_fill_color
...
...
@@ -439,9 +437,9 @@ class PodController:
"profile_sidebar_fill_color"
in
user_dict
and
user_dict
[
"profile_sidebar_fill_color"
]
!=
"None"
):
profile_sidebar_fill_color
=
self
.
handle
_color
(
profile_sidebar_fill_color
=
self
.
add
_color
(
user_dict
[
"profile_sidebar_fill_color"
],
a
ssociated_item
,
a
ccount
,
"profileSidebarFillColor"
,
)
# Add profile_text_color
...
...
@@ -449,11 +447,11 @@ class PodController:
"profile_text_color"
in
user_dict
and
user_dict
[
"profile_text_color"
]
!=
"None"
):
profile_text_color
=
self
.
handle
_color
(
user_dict
[
"profile_text_color"
],
a
ssociated_item
,
"profileTextColor"
,
profile_text_color
=
self
.
add
_color
(
user_dict
[
"profile_text_color"
],
a
ccount
,
"profileTextColor"
,
)
def
handle_new
_follower
(
def
add
_follower
(
self
,
follower_dict
,
account_to_add
,
person_to_add
,
account_to_follow
,
):
"""
...
...
@@ -483,7 +481,7 @@ class PodController:
if
edge
:
return
follower_account
def
handle_existing
_follower
(
self
,
all_accounts
,
account_to_follow
):
def
update
_follower
(
self
,
all_accounts
,
account_to_follow
):
"""
Takes in a list of all_accounts, and account_to_follow.
Checks if existing follower account is following account_to_follow and if not, adds follows edge
...
...
@@ -545,18 +543,16 @@ class PodController:
all_accounts
=
self
.
client
.
search
({
"type"
:
"Account"
})
# If account exists, check that they have an edge to account and if not, add it
if
any
(
item
.
externalId
==
follower_id
for
item
in
all_accounts
):
existing_follower
=
self
.
handle_existing_follower
(
all_accounts
,
account_to_follow
)
existing_follower
=
self
.
update_follower
(
all_accounts
,
account_to_follow
)
return
existing_follower
else
:
follower_from_db
=
self
.
handle_new
_follower
(
follower_from_db
=
self
.
add
_follower
(
follower_id
,
account_to_add
,
person_to_add
,
account_to_follow
,
)
return
follower_from_db
def
handle_hash
tags
(
self
,
tags
,
associated_item
):
def
add_message_
tags
(
self
,
tags
,
associated_item
):
"""
Takes in hashtags and their associated item.
Adds tags to Hashtags table and creates edge with associated item
...
...
@@ -593,7 +589,7 @@ class PodController:
edge_success
=
self
.
client
.
create_edge
(
edge
)
return
hashtag_to_add
def
handle
_tweet_urls
(
self
,
urls
,
associated_item
):
def
add
_tweet_urls
(
self
,
urls
,
associated_item
):
"""
Takes in url and their associated item.
Adds urls to Resource table and creates edge with associated item
...
...
@@ -630,20 +626,20 @@ class PodController:
print
(
edge
)
edge_success
=
self
.
client
.
create_edge
(
edge
)
def
handle
_tweet_media
(
self
,
story
,
associated_item
):
def
add
_tweet_media
(
self
,
tweet
,
associated_item
):
"""
Takes in
story(
tweet
)
dictionary and it's associated item.
Takes in tweet dictionary and it's associated item.
Adds media to IPhoto table and creates edge with associated item
Parameters:
story
(dict): Tweet dictionary. This is got from twitter.
tweet
(dict): Tweet dictionary. This is got from twitter.
associated_item (object): Object instance of the pod item that tagged.
Returns:
"""
# Add media
if
"media"
in
story
[
"entities"
]:
for
item
in
story
[
"entities"
][
"media"
]:
if
"media"
in
tweet
[
"entities"
]:
for
item
in
tweet
[
"entities"
][
"media"
]:
if
item
[
"type"
]
==
"photo"
:
image_from_db
=
self
.
add_image
(
item
[
"media_url_https"
])
edge
=
Edge
(
associated_item
,
image_from_db
,
"uploads"
)
...
...
@@ -718,28 +714,28 @@ class PodController:
edge_success
=
self
.
client
.
create_edge
(
edge
)
return
source_to_add
def
handle_quoted_status
(
self
,
quoted_
status
,
associated_item
):
def
handle_quoted_status
(
self
,
quoted_
msg
,
associated_item
):
"""
Takes in quoted_
status
and it's associated item.
Takes in quoted_
msg
and it's associated item.
Checks if quoted status exists in Message and updates else adds it
Parameters:
quoted_
status
(dict): Symbols list.
quoted_
msg
(dict): Symbols list.
associated_item (object): Object instance of the pod item that tagged.
Returns:
quoted_story_from_db(object): Story of type Message added in pod
"""
all_tweets
=
self
.
client
.
search
({
"type"
:
"Message"
})
if
any
(
item
.
externalId
==
quoted_
status
[
"id_str"
]
for
item
in
all_tweets
):
if
any
(
item
.
externalId
==
quoted_
msg
[
"id_str"
]
for
item
in
all_tweets
):
edges
=
self
.
client
.
get_edges
(
associated_item
.
id
)
if
not
any
(
edge
[
"name"
]
==
"quotes"
and
edge
[
"item"
].
externalId
==
quoted_
status
[
"id_str"
]
and
edge
[
"item"
].
externalId
==
quoted_
msg
[
"id_str"
]
for
edge
in
edges
):
for
item
in
all_tweets
:
if
item
.
externalId
==
quoted_
status
[
"id_str"
]:
if
item
.
externalId
==
quoted_
msg
[
"id_str"
]:
quoted_story_from_db
=
item
edge
=
Edge
(
associated_item
,
quoted_story_from_db
,
"quotes"
)
print
(
edge
)
...
...
@@ -747,124 +743,93 @@ class PodController:
return
quoted_story_from_db
else
:
self
.
add_quoted_
status
(
quoted_
status
,
associated_item
)
self
.
add_quoted_
msg
(
quoted_
msg
,
associated_item
)
def
add_quoted_
status
(
self
,
quoted_
status
,
associated_item
):
def
add_quoted_
msg
(
self
,
quoted_
msg
,
associated_item
):
"""
Takes in quoted_
status
and it's associated item.
Adds quoted_
status
to Message table and creates edge with associated item
Takes in quoted_
msg
and it's associated item.
Adds quoted_
msg
to Message table and creates edge with associated item
Adds symbols, urls, source and hashtags to quoted_story
Parameters:
quoted_
status
(dict): Symbols list.
quoted_
msg
(dict): Symbols list.
associated_item (object): Object instance of the pod item that tagged.
Returns:
quoted_
story
_to_add(object): Story of type Message added in pod
quoted_
msg
_to_add(object): Story of type Message added in pod
"""
quoted_story_to_add
=
Message
(
quoted_status
[
"id_str"
],
quoted_status
[
"full_text"
],
)
quoted_msg_to_add
=
Message
(
quoted_msg
[
"id_str"
],
quoted_msg
[
"full_text"
],)
success
=
self
.
client
.
create
(
quoted_
story
_to_add
)
success
=
self
.
client
.
create
(
quoted_
msg
_to_add
)
if
success
:
# Add quotes edge
edge
=
Edge
(
associated_item
,
quoted_
story
_to_add
,
"quotes"
)
edge
=
Edge
(
associated_item
,
quoted_
msg
_to_add
,
"quotes"
)
print
(
edge
)
edge_success
=
self
.
client
.
create_edge
(
edge
)
# Add quoted status source
if
(
quoted_status
[
"source"
]
is
not
None
and
quoted_status
[
"source"
]
!=
"None"
):
self
.
add_tweet_source
(
quoted_status
[
"source"
],
quoted_story_to_add
)
if
quoted_msg
[
"source"
]
is
not
None
and
quoted_msg
[
"source"
]
!=
"None"
:
self
.
add_tweet_source
(
quoted_msg
[
"source"
],
quoted_msg_to_add
)
# Add hashtags status source
if
"entities"
in
quoted_
status
:
if
"hashtags"
in
quoted_
status
[
"entities"
]:
self
.
handle_hash
tags
(
quoted_
status
[
"entities"
][
"hashtags"
],
quoted_
story
_to_add
,
if
"entities"
in
quoted_
msg
:
if
"hashtags"
in
quoted_
msg
[
"entities"
]:
self
.
add_message_
tags
(
quoted_
msg
[
"entities"
][
"hashtags"
],
quoted_
msg
_to_add
,
)
# Add urls
if
"urls"
in
quoted_
status
[
"entities"
]:
self
.
handle
_tweet_urls
(
quoted_
status
[
"entities"
][
"urls"
],
quoted_
story
_to_add
,
if
"urls"
in
quoted_
msg
[
"entities"
]:
self
.
add
_tweet_urls
(
quoted_
msg
[
"entities"
][
"urls"
],
quoted_
msg
_to_add
,
)
# Add symbols
if
"symbols"
in
quoted_
status
[
"entities"
]:
if
"symbols"
in
quoted_
msg
[
"entities"
]:
self
.
handle_tweet_symbols
(
quoted_
status
[
"entities"
][
"symbols"
],
quoted_
story
_to_add
,
quoted_
msg
[
"entities"
][
"symbols"
],
quoted_
msg
_to_add
,
)
return
quoted_
story
_to_add
return
quoted_
msg
_to_add
def
add_tweet
(
self
,
story
,
story
_to_add
,
current_user
):
def
add_tweet
(
self
,
tweet
,
msg
_to_add
,
current_user
):
"""
Takes in a dictionary of the user, account and person to add and adds account and person for that user
Parameters:
story
(dict): A dictionary of the tweet returned from twitter.
story
_to_add (object): An instance of the Message class for the tweet.
tweet
(dict): A dictionary of the tweet returned from twitter.
msg
_to_add (object): An instance of the Message class for the tweet.
current_user (object): An instance of the Resource class for the tweet.
Returns:
story
_to_add (object): An object of the added account from the pod.
msg
_to_add (object): An object of the added account from the pod.
"""
all_tweets
=
self
.
client
.
search
({
"type"
:
"Message"
})
if
not
any
(
item
.
externalId
==
story
[
"id_str"
]
for
item
in
all_tweets
):
success
=
self
.
client
.
create
(
story
_to_add
)
if
not
any
(
item
.
externalId
==
tweet
[
"id_str"
]
for
item
in
all_tweets
):
success
=
self
.
client
.
create
(
msg
_to_add
)
if
success
:
# Add sender edge
edge
=
Edge
(
current_user
,
story
_to_add
,
"sender"
)
edge
=
Edge
(
current_user
,
msg
_to_add
,
"sender"
)
print
(
edge
)
edge_success
=
self
.
client
.
create_edge
(
edge
)
# Add source
if
story
[
"source"
]
is
not
None
and
story
[
"source"
]
!=
"None"
:
self
.
add_tweet_source
(
story
[
"source"
],
story
_to_add
)
if
tweet
[
"source"
]
is
not
None
and
tweet
[
"source"
]
!=
"None"
:
self
.
add_tweet_source
(
tweet
[
"source"
],
msg
_to_add
)
# Add language
if
story
[
"lang"
]
!=
"None"
:
language_from_db
=
self
.
add_language
(
story
[
"lang"
],
story
_to_add
)
if
"entities"
in
story
:
if
tweet
[
"lang"
]
!=
"None"
:
language_from_db
=
self
.
add_language
(
tweet
[
"lang"
],
msg
_to_add
)
if
"entities"
in
tweet
:
# Add hashtags
if
"hashtags"
in
story
[
"entities"
]:
self
.
handle_hashtags
(
story
[
"entities"
][
"hashtags"
],
story_to_add
)
if
"hashtags"
in
tweet
[
"entities"
]:
self
.
add_message_tags
(
tweet
[
"entities"
][
"hashtags"
],
msg_to_add
)
# Add urls
if
"urls"
in
story
[
"entities"
]:
self
.
handle
_tweet_urls
(
story
[
"entities"
][
"urls"
],
story
_to_add
)
if
"urls"
in
tweet
[
"entities"
]:
self
.
add
_tweet_urls
(
tweet
[
"entities"
][
"urls"
],
msg
_to_add
)
# Add symbols
if
"symbols"
in
story
[
"entities"
]:
if
"symbols"
in
tweet
[
"entities"
]:
self
.
handle_tweet_symbols
(
story
[
"entities"
][
"symbols"
],
story
_to_add
tweet
[
"entities"
][
"symbols"
],
msg
_to_add
)
# Add images and videos(videos are a future plan)
# Currently cannot upload files to POD with pluginAuth
# self.handle_tweet_media(
story, story
_to_add)
if
"quoted_status"
in
story
and
story
[
"quoted_status"
]
!=
None
:
# self.handle_tweet_media(
tweet_dict, msg
_to_add)
if
"quoted_status"
in
tweet
and
tweet
[
"quoted_status"
]
!=
None
:
quoted_story_from_db
=
self
.
handle_quoted_status
(
story
[
"quoted_status"
],
story
_to_add
tweet
[
"quoted_status"
],
msg
_to_add
)
return
story_to_add
def
get_user_max_tweet_id
(
self
,
sender_id
):
"""
Takes in a sender id of the tweets, and returns the maximum id found among the specified user's tweets
Parameters:
sender_id (str): An id_str of the user.
Returns:
max_id (str): The maximum id of the existing user tweets.
"""
all_accounts
=
self
.
client
.
search
({
"type"
:
"Account"
})
max_id
=
1
for
existing_account
in
all_accounts
:
edges
=
self
.
client
.
get_edges
(
existing_account
.
id
)
for
edge
in
edges
:
if
(
edge
[
"name"
]
==
"sender"
and
existing_account
.
externalId
==
sender_id
):
if
int
(
edge
[
"item"
].
externalId
)
>
max_id
:
max_id
=
int
(
edge
[
"item"
].
externalId
)
return
max_id
return
msg_to_add
This diff is collapsed.
Click to expand it.
twitter_importer/twitter_plugin.py
+
27
-
6
View file @
5ec6ee05
...
...
@@ -34,7 +34,7 @@ class TwitterPlugin(PluginFlow):
self
.
client
=
client
self
.
twitter_client
=
TwitterClient
()
self
.
pod_controller
=
PodController
(
client
)
self
.
followings_tweet_count
=
1
# These can be limited differently
self
.
followings_tweet_count
=
5
# These can be limited differently
self
.
isTest
=
isTest
self
.
user_account
=
None
self
.
user_external_id
=
None
...
...
@@ -57,7 +57,7 @@ class TwitterPlugin(PluginFlow):
"""
Imports user data, followers, friends and user timeline
"""
max_amount
=
1
if
self
.
isTest
else
None
max_amount
=
5
if
self
.
isTest
else
None
self
.
setup_api
()
user
=
self
.
twitter_client
.
api
.
me
()
# Set username for authenticated user and limit of data to import
...
...
@@ -170,7 +170,7 @@ class TwitterPlugin(PluginFlow):
"""
# Timeline tweets
print
(
"Fetching user tweets..."
)
max_id
=
self
.
pod_controller
.
get_user_max_tweet_id
(
self
.
user_external_id
)
max_id
=
self
.
get_user_max_tweet_id
(
self
.
user_external_id
)
timeline_tweets
=
self
.
twitter_client
.
get_timeline
(
max_id
)
if
timeline_tweets
is
not
None
:
self
.
handle_timeline
(
timeline_tweets
,
current_user_account
)
...
...
@@ -235,9 +235,7 @@ class TwitterPlugin(PluginFlow):
print
(
current_follower
)
# Fetch following tweets from the last id saved
print
(
"Fetching following's tweets..."
)
max_id
=
self
.
pod_controller
.
get_user_max_tweet_id
(
current_item
[
"id_str"
]
)
max_id
=
self
.
get_user_max_tweet_id
(
current_item
[
"id_str"
])
following_timeline
=
self
.
twitter_client
.
get_timeline
(
max_id
+
1
,
self
.
followings_tweet_count
,
...
...
@@ -320,6 +318,29 @@ class TwitterPlugin(PluginFlow):
datetime_now
=
int
(
new_datetime
.
strftime
(
"%Y%m%d%H%M%S"
))
return
datetime_now
def
get_user_max_tweet_id
(
self
,
sender_id
):
"""
Takes in a sender id of the tweets, and returns the maximum id found among the specified user's tweets
Parameters:
sender_id (str): An id_str of the user.
Returns:
max_id (str): The maximum id of the existing user tweets.
"""
all_accounts
=
self
.
client
.
search
({
"type"
:
"Account"
})
max_id
=
1
for
existing_account
in
all_accounts
:
edges
=
self
.
client
.
get_edges
(
existing_account
.
id
)
for
edge
in
edges
:
if
(
edge
[
"name"
]
==
"sender"
and
existing_account
.
externalId
==
sender_id
):
if
int
(
edge
[
"item"
].
externalId
)
>
max_id
:
max_id
=
int
(
edge
[
"item"
].
externalId
)
return
max_id
def
add_to_schema
(
self
):
self
.
client
.
add_to_schema
(
Account
(
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment