fix: Improved importing of duplicate posts and comments
All checks were successful
Build / build (push) Successful in 4m10s

This commit is contained in:
Daniel Kempkens 2023-10-20 18:23:07 +02:00
parent d7bf17fa34
commit 65f8468daa
Signed by: daniel
SSH key fingerprint: SHA256:Ks/MyhQYcPRQiwMKLAKquWCdCPe3JXlb1WttgnAoSeM
3 changed files with 64 additions and 40 deletions

View file

@ -43,6 +43,15 @@ defmodule BdfrBrowser.Comment do
)
end
def get_import(id) do
from(c in __MODULE__,
select: %{
id: c.id
},
where: c.id == ^id
)
end
def search(str), do: search(str, nil)
def search(str, subreddits) when is_nil(subreddits) do

View file

@ -310,7 +310,9 @@ defmodule BdfrBrowser.Importer do
defp import_post(post, subreddit) when not is_nil(subreddit) do
id = post["id"]
db_post = id |> Post.get_import() |> Repo.one()
if is_nil(db_post) do
%Post{
id: id,
title: post["title"],
@ -323,16 +325,18 @@ defmodule BdfrBrowser.Importer do
filename: Path.basename(post["filename"], ".json"),
subreddit: subreddit
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
|> Repo.insert()
else
{:ok, db_post}
end
end
defp import_comment(comment, post, parent) when not is_nil(post) do
id = comment["id"]
db_comment = id |> Comment.get_import() |> Repo.one()
{:ok, parent} =
if is_nil(db_comment) do
%Comment{
id: id,
author: comment["author"],
@ -342,10 +346,10 @@ defmodule BdfrBrowser.Importer do
post: post,
parent: parent
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
|> Repo.insert()
else
{:ok, db_comment}
end
children = for child <- comment["replies"], do: import_comment(child, post, parent)
@ -370,8 +374,10 @@ defmodule BdfrBrowser.Importer do
id = calculate_message_id(message, chat.id)
message_content = message["content"]["Message"]
{:ok, posted_at, 0} = DateTime.from_iso8601(message["timestamp"])
db_message = Repo.get(Message, id)
{:ok, message_record} =
if is_nil(db_message) do
%Message{
id: id,
author: message["author"],
@ -379,10 +385,10 @@ defmodule BdfrBrowser.Importer do
posted_at: posted_at,
chat: chat
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
|> Repo.insert()
else
{:ok, db_message}
end
existing_image =
message_record.message == "Image" or

View file

@ -98,6 +98,15 @@ defmodule BdfrBrowser.Post do
having(query, [p, c, s], count(c.id) > ^more_than)
end
def get_import(id) do
from(p in __MODULE__,
select: %{
id: p.id
},
where: p.id == ^id
)
end
def get_full(id) do
from(p in __MODULE__,
where: p.id == ^id,