feat: added albums api and duplication

flickr_plugin/config.py

 import os
+
 from dotenv import load_dotenv
 
 load_dotenv()

flickr_plugin/data/schema.py

-from pymemri.data.schema import Item, Person, Account, OauthFlow, Edge, Location, Hashtag
+from hashlib import sha256
+from typing import List, Optional
+
+from pymemri.data.schema import (Account, Edge, Hashtag, Item, Location,
+                                 OauthFlow, Person)
 from pymemri.data.schema.photo import Photo
 
-from typing import Optional, List
-from hashlib import sha256
 
 
 class OauthFlow(OauthFlow):
@@ -72,14 +74,31 @@ class Favourite(Item):
     photo: List[Photo] = None
 
 
-class Service(Item):
-    name: Optional[str] = ""
+class Person(Person):
     # Edges
+    owner: List[Gallery] = None
+    owns: List[Photo] = None
+
+
+class Album(Item):
+    username: Optional[str] = ""
+    primary: Optional[str] = ""
+    secret: Optional[str] = ""
+    server: Optional["str"] = ""
+    farm: Optional[int] = None
+    count_photos: Optional[int] = 0
+    count_videos: Optional[int] = 0
+    count_comments: Optional[int] = None
+    count_views: Optional[int] = 0
+    description: Optional[str] = ""
+    title: Optional[str] = ""
     photo: List[Photo] = None
-    gallery: List[Gallery] = None
 
 
-class Person(Person):
+class Service(Item):
+    name: Optional[str] = ""
     # Edges
-    owner: List[Gallery] = None
-    owns: List[Photo] = None
+    photo: List[Photo] = None
+    gallery: List[Gallery] = None
+    album: List[Album] = None
+    favourite: List[Favourite] = None

flickr_plugin/plugin.py

+from concurrent.futures import ThreadPoolExecutor
+from time import sleep, time
+
+from flickrapi.auth import FlickrAccessToken
 from loguru import logger
 from pymemri.plugin.pluginbase import PluginBase
 from .data.schema import (
@@ -36,7 +40,14 @@ class FlickrPlugin(PluginBase):
     writing = False
     check_person = False
 
-    existing_item_ids = {"Account": set(), "Person": set(), "Photo": set(), "Country": dict(), "County": dict()}
+    existing_item_ids = {
+        "Account": set(),
+        "Person": set(),
+        "Photo": set(),
+        "Country": dict(),
+        "County": dict(),
+        "Album": set(),
+    }
 
     def get_external_ids(self, itemType: str):
         external_ids = []
@@ -186,9 +197,14 @@ class FlickrPlugin(PluginBase):
         self.process_photos(photos)
 
     def process_favourite_photo_from_user(self, photos):
-        favourite = Favourite(service_name=SERVICE_NAME)
-        self.client.create(favourite)
-        self.process_photos(photos, favourite)
+        if self.flickr_item.favourite:
+            favourite = self.flickr_item.favourite[0]
+        else:
+            favourite = Favourite(service_name=SERVICE_NAME, externalId=SERVICE_NAME)
+            self.client.bulk_action(
+                create_items=[favourite], create_edges=[Edge(self.flickr_item, favourite, "favourite")]
+            )
+        self.process_photos(photos, favourite=favourite)
 
     def process_galleries_photo_from_user(self, galleries):
         for gallery_vals in galleries:
@@ -205,25 +221,67 @@ class FlickrPlugin(PluginBase):
                     count_views=gallery_vals["count_views"],
                     count_comments=gallery_vals["count_comments"],
                 )
-            self.client.bulk_action(create_items=[gallery], create_edges=[Edge(self.flickr_item, gallery, "gallery")])
-            self.process_photos(gallery_vals["photos"], gallery)
+                self.client.bulk_action(
+                    create_items=[gallery], create_edges=[Edge(self.flickr_item, gallery, "gallery")]
+                )
+            self.process_photos(gallery_vals["photos"], gallery=gallery)
 
-    def process_photos(self, photos, gallery=None, favourite=None):
+    def process_albums_photo_from_user(self, albums):
+        for album_vals in albums:
+            if album_vals["id"] in self.existing_item_ids["Album"]:
+                album = self.get_or_create_item(album_vals["id"], Album)
+            else:
+                album = Album(
+                    externalId=album_vals["id"],
+                    username=album_vals["username"],
+                    primary=album_vals["primary"],
+                    secret=album_vals["secret"],
+                    server=album_vals["server"],
+                    farm=album_vals["farm"],
+                    count_views=int(album_vals.get("count_views", 0)),
+                    count_comments=int(album_vals.get("count_comments", 0)),
+                    count_photos=album_vals["count_photos"],
+                    count_videos=album_vals["count_videos"],
+                    title=album_vals["title"]["_content"],
+                    description=album_vals["description"]["_content"],
+                )
+
+                self.client.bulk_action(create_items=[album], create_edges=[Edge(self.flickr_item, album, "album")])
+            self.process_photos(album_vals["photos"], album=album)
+
+    def process_photos(self, photos, gallery=None, favourite=None, album=None):
         with ThreadPoolExecutor(max_workers=10) as executor:
             for photo in photos:
-                executor.submit(self.process_photo, photo, gallery, favourite)
+                executor.submit(self.process_photo, photo, gallery, favourite, album)
                 # self.process_photo(photo, gallery, favourite)
 
         executor.shutdown()
 
-    def process_photo(self, photo, gallery=None, favourite=None):
+    def process_photo(self, photo, gallery=None, favourite=None, album=None):
         create_items = []
         create_edges = []
-        if photo["id"] in self.existing_item_ids["Photo"]:
+        if photo["id"] in self.get_external_ids(itemType="Photo"):
+            p = self.get_or_create_item(photo["id"], Photo)
+            if gallery:
+                if p not in gallery.photo:
+                    create_edges.append(Edge(gallery, p, "photo"))
+
+            if favourite:
+                if p not in album.photo:
+                    create_edges.append(Edge(favourite, p, "photo"))
+
+            if album:
+                if p not in album.photo:
+                    create_edges.append(Edge(album, p, "photo"))
+
+            if create_edges:
+                self.client.bulk_action(create_edges=create_edges)
+
             return
 
         try:
             p = Photo.from_bytes(self.flickr_service.download_photo(photo["id"]))
+            p.externalId = photo["id"]
             if "location" in photo:
                 location, country, county, region = self.process_location(photo["location"])
                 create_items.append(location)
@@ -270,6 +328,9 @@ class FlickrPlugin(PluginBase):
             if favourite:
                 create_edges.append(Edge(favourite, p, "photo"))
 
+            if album:
+                create_edges.append(Edge(album, p, "photo"))
+
             # self.add_to_queue(
             #     items=create_items,
             #     edges=create_edges,
@@ -330,6 +391,7 @@ class FlickrPlugin(PluginBase):
             user_picture = Photo.from_bytes(user_picture)
             user_picture.caption = "Profile picture"
             user_picture.name = information["realname"]["_content"]
+            user_picture.externalId = information["id"]
 
             edges.append(Edge(user_picture, user_picture.file[0], "file"))
             edges.append(Edge(user_account, user_picture, "profilePicture"))
@@ -358,6 +420,7 @@ class FlickrPlugin(PluginBase):
             Camera,
             Favourite,
             Hashtag,
+            Album,
         )
 
     def run(self):
@@ -373,6 +436,7 @@ class FlickrPlugin(PluginBase):
         self.existing_item_ids["Country"] = self.get_location(itemType="Country")
         self.existing_item_ids["County"] = self.get_location(itemType="County")
         self.existing_item_ids["Region"] = self.get_location(itemType="Region")
+        self.existing_item_ids["Album"] = self.get_external_ids(itemType="Album")
 
         brands = self.flickr_service.api_brand()
         self.process_brand(brands)
@@ -387,6 +451,7 @@ class FlickrPlugin(PluginBase):
         self.process_galleries_photo_from_user(self.flickr_service.get_user_gallery())
         self.process_photo_from_user(self.flickr_service.get_user_photo())
         self.process_favourite_photo_from_user(self.flickr_service.get_user_photo_favourite())
+        self.process_albums_photo_from_user(self.flickr_service.get_user_albums())
         # self.write()
 
     def add_to_queue(self, items=None, updates=None, edges=None, files=None):

flickr_plugin/service/flickr_api.py

-from flickrapi import FlickrAPI
 import json
-from flickr_plugin.config import API_KEY, SECRET_KEY
 import requests
 
+from flickrapi import FlickrAPI
+
+from Flickr_plugin.config import API_KEY, SECRET_KEY
+
+
 
 def json_parse(func):
     def decorator(self, *args, **kwargs):
@@ -201,6 +204,23 @@ class FlickrService:
 
         return res
 
+    def get_user_photo_from_albums(self, album_id):
+        res = []
+        page = None
+        while True:
+            photos = self.api_photo_from_album(album_id=album_id, page=page)["photoset"]
+            for photo in photos["photo"]:
+                photo = self.update_addtional_information_photo(photo)
+
+            res.extend(photos["photo"])
+            page = photos["page"]
+            pages = photos["pages"]
+            if page == pages:
+                break
+            page += 1
+
+        return res
+
     def get_user_gallery(self, user_id="me"):
         res = []
         page = None
@@ -216,3 +236,19 @@ class FlickrService:
             page += 1
 
         return res
+
+    def get_user_albums(self, user_id="me"):
+        res = []
+        page = None
+        while True:
+            albums = self.api_albums_from_user(page=None)["photosets"]
+            for album in albums["photoset"]:
+                album["photos"] = self.get_user_photo_from_albums(album_id=album["id"])
+            res.extend(albums["photoset"])
+            page = albums["page"]
+            pages = albums["pages"]
+            if page == pages:
+                break
+            page += 1
+
+        return res