partage/systeme-retro-contenu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🚀 Add feature: Charging in Milvus is now working

Browse source
This commit is contained in:
François Pelletier 2025-05-21 17:21:57 -04:00
parent f9e5a6e013
commit 64832e2989
23 changed files with 354 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

6
backend/app/convert/convert_facebook_business_posts_json.py
View file

@ -16,9 +16,9 @@ class FacebookBusinessPostsConverter(BaseConverter):
posts_medias = []
for post in self.datadict:
data_post_items = post['data']
texte_post_list = [item['post'] for item in data_post_items if
content_post_list = [item['post'] for item in data_post_items if
item.get('post')]
texte = "\n".join(texte_post_list)
content = "\n".join(content_post_list)
for attachment in post['attachments']:
if attachment.get('data'):
@ -27,7 +27,7 @@ class FacebookBusinessPostsConverter(BaseConverter):
media = data_item['media']
posts_medias.append({
"chemin": [media["uri"]],
"texte": texte,
"content": content,
"creation_timestamp": media[
"creation_timestamp"]
})

5
backend/app/convert/convert_facebook_comments_json.py
View file

@ -4,9 +4,8 @@ from typing import Union, List, Dict
import pandas as pd
from app.config import logger
from app.models import ConversionResponse
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
from app.convert.utils.convert_encoding_meta import convert_encoding_meta
@ -31,7 +30,7 @@ class FacebookCommentsConverter(BaseConverter):
if data_item.get('comment'):
comment_data = data_item['comment']
facebook_comments.append({
"texte": comment_data["comment"],
"content": comment_data["comment"],
"creation_timestamp": comment_data["timestamp"]
})

7
backend/app/convert/convert_facebook_posts_json.py
View file

@ -1,12 +1,11 @@
import json
from typing import Union, List, Dict
import pandas as pd
import json
from app.config import logger
from app.models import ConversionResponse
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
from app.convert.utils.convert_encoding_meta import convert_encoding_meta
@ -31,7 +30,7 @@ class FacebookPostsConverter(BaseConverter):
def rename_columns(self) -> None:
self.df.rename(columns={
"description": "texte",
"description": "content",
"uri": "chemin"
}, inplace=True)

7
backend/app/convert/convert_instagram_comments_json.py
View file

@ -4,10 +4,9 @@ from typing import Union, List, Dict
import pandas as pd
from app.config import logger
from app.convert.utils.convert_encoding_meta import convert_encoding_meta
from app.models import ConversionResponse
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
from app.convert.utils.convert_encoding_meta import convert_encoding_meta
class InstagramCommentsConverter(BaseConverter):
@ -29,7 +28,7 @@ class InstagramCommentsConverter(BaseConverter):
[]) + self.datadict.get(
'post_comments_1', []):
ig_comments.append({
"texte": comment['string_map_data']['Comment']['value'],
"content": comment['string_map_data']['Comment']['value'],
'creation_timestamp': int(
comment['string_map_data']['Time']['timestamp']),
'index': self.df.index,

4
backend/app/convert/convert_instagram_posts_json.py
View file

@ -19,7 +19,7 @@ class InstagramPostsConverter(BaseConverter):
media = medias[0]
posts_medias.append({
"chemin": [media["uri"]],
"texte": media["title"],
"content": media["title"],
"creation_timestamp": media["creation_timestamp"]
})
else:
@ -28,7 +28,7 @@ class InstagramPostsConverter(BaseConverter):
list_uris = [media['uri'] for media in medias]
posts_medias.append({
"chemin": list_uris,
"texte": title,
"content": title,
"creation_timestamp": creation_timestamp
})

7
backend/app/convert/convert_instagram_reels_json.py
View file

@ -1,13 +1,12 @@
import json
import datetime
import json
from typing import Union, List, Dict
import pandas as pd
from app.config import logger
from app.models import ConversionResponse
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
from app.convert.utils.convert_encoding_meta import convert_encoding_meta
@ -34,7 +33,7 @@ class InstagramReelsConverter(BaseConverter):
def rename_columns(self) -> None:
self.df.rename(columns={
"title": "texte",
"title": "content",
"uri": "chemin"
}, inplace=True)

7
backend/app/convert/convert_instagram_stories_json.py
View file

@ -1,13 +1,12 @@
import json
import datetime
import json
from typing import Dict, Union, List
import pandas as pd
from app.config import logger
from app.models import ConversionResponse
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
from app.convert.utils.convert_encoding_meta import convert_encoding_meta
@ -33,7 +32,7 @@ class InstagramStoriesConverter(BaseConverter):
def rename_columns(self) -> None:
self.df.rename(columns={
"title": "texte",
"title": "content",
"uri": "chemin"
}, inplace=True)

10
backend/app/convert/convert_linkedin_comments_csv.py
View file

@ -1,11 +1,11 @@
import pandas as pd
import datetime
from typing import Dict, Union, List
import pandas as pd
from app.config import logger
from app.models import ConversionResponse
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
class LinkedInCommentsConverter(BaseConverter):
@ -37,11 +37,11 @@ class LinkedInCommentsConverter(BaseConverter):
def rename_columns(self) -> None:
self.df.rename(columns={
"Link": "url",
"Message": "texte"
"Message": "content"
}, inplace=True)
def clean_data(self) -> None:
self.df["texte"] = self.df["texte"].apply(lambda x: str(x))
self.df["content"] = self.df["content"].apply(lambda x: str(x))
self.df["chemin"] = ""
self.df.fillna(value="", inplace=True)

8
backend/app/convert/convert_linkedin_shares_csv.py
View file

@ -3,8 +3,8 @@ from typing import Dict, Union, List
import pandas as pd
from app.convert.utils.content_from_file import content_from_file
from app.convert.base_converter import BaseConverter
from app.convert.utils.content_from_file import content_from_file
class LinkedInSharesConverter(BaseConverter):
@ -26,7 +26,7 @@ class LinkedInSharesConverter(BaseConverter):
def rename_columns(self) -> None:
self.df = self.df.rename(columns={
"ShareLink": "uri",
"ShareCommentary": "texte",
"ShareCommentary": "content",
"Date": "creation_timestamp"
})
@ -34,8 +34,8 @@ class LinkedInSharesConverter(BaseConverter):
"""Clean and preprocess the DataFrame."""
self.df = self.df.fillna("")
self.df = self.df.drop_duplicates(
subset=["texte", "creation_timestamp"])
self.df = self.df[self.df["texte"].str.strip() != ""]
subset=["content", "creation_timestamp"])
self.df = self.df[self.df["content"].str.strip() != ""]
def convert_linkedin_shares_csv(content: Union[str, bytes]) -> List[Dict]: